From 2ec081e80f79904724afc07c90f18fc788cd23f5 Mon Sep 17 00:00:00 2001 From: shiyi Date: Mon, 1 Jul 2024 17:55:23 +0800 Subject: [PATCH] =?UTF-8?q?package=E9=83=A8=E5=88=86=E9=87=8D=E6=96=B0?= =?UTF-8?q?=E7=BB=84=E7=BB=87=EF=BC=9B=E5=86=97=E4=BD=99=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/htfp/weather/config/Config.java | 18 ---- .../weather/download/BaseDataDownloader.java | 13 +-- .../download/{ => gfs}/GfsDataConfig.java | 4 +- .../gfs}/GfsDownloadVariableEnum.java | 2 +- .../download/{ => gfs}/GfsDownloader.java | 27 ++---- .../{info => download/gfs}/GfsLevelsEnum.java | 3 +- .../gfs}/GfsVariableHeightEnum.java | 4 +- .../gfs}/GfsVariableIsobaricEnum.java | 4 +- .../weather/griddata/common/TableConfig.java | 2 +- .../griddata/common/TableConfigStatic.java | 2 +- .../{utils => common}/ValueRange.java | 4 +- .../griddata/operation/GfsDataFetcher.java | 18 +++- .../griddata/operation/GfsDataImport.java | 51 ++++++---- .../griddata/operation/IDataFetch.java | 9 -- .../weather/schedule/GridDataProcessor.java | 12 +-- .../com/htfp/weather/utils/DateTimeUtils.java | 2 +- .../htfp/weather/utils/HttpClientUtils.java | 5 +- .../web/controller/ConfigController.java | 4 +- .../controller/SurfaceWeatherController.java | 10 +- .../controller/UpperWeatherController.java | 39 +++++--- .../web/pojo/request/GfsConfigUpdate.java | 2 +- .../web/pojo/request/PlaneRequest.java | 17 ++-- .../web/pojo/request/ProfileRequest.java | 1 - .../web/pojo/response/ProfileDataset.java | 42 ++++++++ .../web/service/GfsDataServiceImpl.java | 95 +++++++++++++------ .../service/surfaceapi/CaiYunServiceImpl.java | 6 +- .../service/surfaceapi/HeFengServiceImpl.java | 15 +-- .../weather/download/GfsDataConfigTest.java | 3 +- .../weather/download/GfsDownloaderTest.java | 4 +- .../operation/GfsDataFetcherTest.java | 3 +- .../weather/griddata/utils/GfsUtilsTest.java | 30 ------ 31 files changed, 249 insertions(+), 202 deletions(-) delete mode 100644 weather-service/src/main/java/com/htfp/weather/config/Config.java rename weather-service/src/main/java/com/htfp/weather/download/{ => gfs}/GfsDataConfig.java (97%) rename weather-service/src/main/java/com/htfp/weather/{info => download/gfs}/GfsDownloadVariableEnum.java (96%) rename weather-service/src/main/java/com/htfp/weather/download/{ => gfs}/GfsDownloader.java (93%) rename weather-service/src/main/java/com/htfp/weather/{info => download/gfs}/GfsLevelsEnum.java (95%) rename weather-service/src/main/java/com/htfp/weather/{griddata/common => download/gfs}/GfsVariableHeightEnum.java (94%) rename weather-service/src/main/java/com/htfp/weather/{griddata/common => download/gfs}/GfsVariableIsobaricEnum.java (94%) rename weather-service/src/main/java/com/htfp/weather/griddata/{utils => common}/ValueRange.java (85%) delete mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java delete mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java diff --git a/weather-service/src/main/java/com/htfp/weather/config/Config.java b/weather-service/src/main/java/com/htfp/weather/config/Config.java deleted file mode 100644 index 33979cc..0000000 --- a/weather-service/src/main/java/com/htfp/weather/config/Config.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.htfp.weather.config; - -import com.htfp.weather.griddata.common.TableConfig; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; - -/** - * @Author : shiyi - * @Date : 2024/1/22 19:02 - * @Description : - */ -@Component("config") -public class Config { - @Resource - TableConfig tableConfig; - -} diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java index 73fb052..633eef9 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java +++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java @@ -8,8 +8,9 @@ import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.util.List; -/** - * @author shiyi +/** @Author : shiyi + * @Date : 2024/2/28 13:53 + * @Description : GFS数据下载类 */ @Data @ToString @@ -36,16 +37,10 @@ public abstract class BaseDataDownloader { /** * 下载所有目标文件 - * - * @param filesInfo + * @param fileInfoList * @return */ public abstract List downloadAll(List fileInfoList); - - public BaseDataDownloader() { - - } - } diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java similarity index 97% rename from weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java index 5b9a964..2d21a20 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java @@ -1,8 +1,6 @@ -package com.htfp.weather.download; +package com.htfp.weather.download.gfs; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.htfp.weather.info.GfsDownloadVariableEnum; -import com.htfp.weather.info.GfsLevelsEnum; import com.htfp.weather.utils.JSONUtils; import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.ErrorCode; diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java similarity index 96% rename from weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java index 954d5a2..b546ba6 100644 --- a/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java @@ -1,4 +1,4 @@ -package com.htfp.weather.info; +package com.htfp.weather.download.gfs; /** * @Author : shiyi diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java similarity index 93% rename from weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java index 1304f76..7eb02a0 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java @@ -1,10 +1,11 @@ -package com.htfp.weather.download; +package com.htfp.weather.download.gfs; +import com.htfp.weather.download.BaseDataDownloader; +import com.htfp.weather.download.FileInfo; import com.htfp.weather.info.Constant; import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.utils.HttpClientUtils; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; @@ -13,12 +14,7 @@ import ucar.nc2.NetcdfFiles; import javax.annotation.Resource; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.net.ConnectException; -import java.net.URL; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.*; @@ -30,6 +26,7 @@ import java.util.concurrent.Future; /** * @author shiyi + * @ */ @Slf4j @Component("gfsDownloader") @@ -42,7 +39,6 @@ public class GfsDownloader extends BaseDataDownloader { private OffsetDateTime realUpdateDateTime; public GfsDownloader(GfsDataConfig gfsDataConfig) { - super(); this.gfsDataConfig = gfsDataConfig; } @@ -58,8 +54,8 @@ public class GfsDownloader extends BaseDataDownloader { realUpdateDateTime = getAvailableUpdateDateTime(utcStartTime); this.setRefTime(realUpdateDateTime); this.setRefTimeStr(realUpdateDateTime.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))); - // 起报点和实际请求时间的差异 + // 起报点和实际请求时间的差异 this.hourDiff = (int) Duration.between(realUpdateDateTime, utcStartTime).toHours(); log.info("[GFS Download] 起始为当前本地时间：{}", this.getStartTime()); log.info("[GFS Download] GFS最近更新时间UTC：{}，本地时间: {}", realUpdateDateTime, DateTimeUtils.getLocalZoneDateTime(realUpdateDateTime)); @@ -69,24 +65,20 @@ public class GfsDownloader extends BaseDataDownloader { public FileInfo download(FileInfo fileInfo) { return download0(fileInfo, 2); } - private FileInfo download0(FileInfo fileInfo, int retryNum) { + private FileInfo download0(FileInfo fileInfo, int retryNum) { String url = fileInfo.getUrl(); File destDir = new File(fileInfo.getSavePath()); File fileOut = new File(destDir, fileInfo.getFileName()); log.info("[GFS Download] 文件下载中，保存至 {}", fileOut); try { - // TODO 2024/5/24: 增加重试机制 - // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) { - // throw new RuntimeException(); - // } - // FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000); - // FIXME 2024/6/8: 如果连接是https，jar包下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error + // DONE: 改用okhttp FIXME 2024/6/8: 如果连接是https，jar包启动下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error HttpClientUtils.downloadFileByUrl(url, fileOut.getPath()); log.info("[GFS Download] 文件下载成功: {}", fileOut); fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath())); } catch (Exception e) { + // DONE 2024/5/24: 文件服务器在外网不稳定，增加重试机制 fileInfo.setDownloadSuccess(false); if (retryNum > 0) { log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut); @@ -99,6 +91,7 @@ public class GfsDownloader extends BaseDataDownloader { return fileInfo; } + /** 简单校验验证文件完整性，并生成索引文件*/ private boolean fileValid(String file) { try (NetcdfFile ncFile = NetcdfFiles.open(file)) { ncFile.getLocation(); @@ -274,7 +267,7 @@ public class GfsDownloader extends BaseDataDownloader { for (String variable : variables) { stringJoiner.add(String.format("var_%s=on", variable)); } - stringJoiner.add("lev_surface=on"); // gust需要 + stringJoiner.add("lev_surface=on"); // gust, prate需要 return stringJoiner.toString(); } diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java similarity index 95% rename from weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java index bdb7243..66b4c6e 100644 --- a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java @@ -1,4 +1,4 @@ -package com.htfp.weather.info; +package com.htfp.weather.download.gfs; /** * @Author : shiyi @@ -8,6 +8,7 @@ package com.htfp.weather.info; public enum GfsLevelsEnum { // SURFACE(9999, "地面"), + UPPER(0, "高空"), PRES_1000hPa(1000, "1000 hPa"), PRES_975hPa(975, "975 hPa"), PRES_950hPa(950, "950 hPa"), diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java similarity index 94% rename from weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java index d769374..75feacc 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java @@ -1,4 +1,4 @@ -package com.htfp.weather.griddata.common; +package com.htfp.weather.download.gfs; import java.util.ArrayList; import java.util.List; @@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * @Author : shiyi * @Date : 2024/1/15 14:54 - * @Description : 文件变量名和表中变量名的映射 + * @Description : 文件高度坐标变量名和表中变量名的映射 */ public enum GfsVariableHeightEnum { // diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java similarity index 94% rename from weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java index defb295..6051dec 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java +++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java @@ -1,4 +1,4 @@ -package com.htfp.weather.griddata.common; +package com.htfp.weather.download.gfs; import java.util.ArrayList; import java.util.List; @@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * @Author : shiyi * @Date : 2024/1/15 14:54 - * @Description : 文件变量名和表中变量名的映射 + * @Description : 文件气压坐标变量名和表中变量名的映射 */ public enum GfsVariableIsobaricEnum { // diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java index 4f9e376..e5e7991 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java @@ -2,7 +2,7 @@ package com.htfp.weather.griddata.common; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDataConfig; import com.htfp.weather.utils.JSONUtils; import lombok.Data; import lombok.extern.slf4j.Slf4j; diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java index 607e605..bcf7b63 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java @@ -1,7 +1,7 @@ package com.htfp.weather.griddata.common; import com.google.gson.Gson; -import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDataConfig; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java similarity index 85% rename from weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java rename to weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java index aa8ae90..b0c05bb 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java @@ -1,9 +1,11 @@ -package com.htfp.weather.griddata.utils; +package com.htfp.weather.griddata.common; import lombok.Getter; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; /** * @Author : shiyi diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java index c9a30b1..9c4a5f7 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java @@ -7,8 +7,8 @@ import com.aliyun.tablestore.grid.model.GridDataSet; import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.aliyun.tablestore.grid.model.grid.Range; import com.htfp.weather.griddata.common.TableConfig; -import com.htfp.weather.griddata.utils.ValueRange; -import com.htfp.weather.info.GfsLevelsEnum; +import com.htfp.weather.griddata.common.ValueRange; +import com.htfp.weather.download.gfs.GfsLevelsEnum; import com.htfp.weather.utils.NdArrayUtils; import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.ErrorCode; @@ -154,7 +154,19 @@ public class GfsDataFetcher extends BaseTableOperation { Array array = gridDataSet.getVariable(variableName).toArray(); return array; } - + public GridDataSet getProfileByPressure(List variableNameList, OffsetDateTime targetTime, double latitude, double longitude) throws Exception { + lastGridDataSetMeta = getLastGridDataSetMeta(); + GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(lastGridDataSetMeta); + int iTime = getTargetTimeIndex(targetTime); + int iLat= getLatitudeIndex(latitude); + int iLon = getLongitudeIndex(longitude); + int[] origin = new int[]{iTime, 0, iLat, iLon}; + int[] shape = new int[]{1,tableConfig.levSize, 1, 1}; + // TODO 2024/6/17: + fetcher.setVariablesToGet(variableNameList); + fetcher.setOriginShape(origin, shape); + return fetcher.fetch(); + } /** * 获取指定变量在某经纬高处的预报序列 * @param dataSetId 数据集ID diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java index 4ecfd8a..a640cdc 100644 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java @@ -5,8 +5,9 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.aliyun.tablestore.grid.model.StoreOptions; import com.aliyun.tablestore.grid.model.grid.Grid2D; -import com.htfp.weather.griddata.common.GfsVariableHeightEnum; -import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; +import com.htfp.weather.download.FileInfo; +import com.htfp.weather.download.gfs.GfsVariableHeightEnum; +import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum; import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.info.Constant; import com.htfp.weather.utils.MeteoUtils; @@ -32,6 +33,8 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; /** * @author shi_y @@ -85,57 +88,64 @@ public class GfsDataImport extends BaseTableOperation { * @throws Exception */ public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception { - meta.addAttribute("status", "DONE"); tableStoreGrid.updateDataSetMeta(meta); return meta; } public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception { - meta.addAttribute("status", "DONE"); tableStoreGrid.putDataSetMeta(meta); return meta; } + /** 导入指定起报时间的所有文件*/ public List importData(OffsetDateTime refTime) throws Exception { + importing = true; long start = System.currentTimeMillis(); List fileList = getFiles(refTime); if (CollectionUtils.isEmpty(fileList)) { throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES); } log.info("[tablestore] 数据导入开始, refTime = {}...", refTime); + // datasetId和起报时间绑定 String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING)); List fileVariables = getFileVariables(tableConfig.variableList); int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize}; GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape); - List forecastHours = new ArrayList<>(); // 记录到数据库属性中 - - // done 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长 - // done 2024/5/13: 优化方案，使用多线程，并使用Future来获取结果，但是必须保证每个时刻都成功导入 + List forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中 + // todo 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长 List finishedList; - try { + // List> futures = new ArrayList<>(); finishedList = new ArrayList<>(); for (int i = 0; i < fileList.size(); i++) { - // for(int i = 0; i < 200; i++) { String file = fileList.get(i); int iTime = i; int forecastHour = getForecastHourFromFilename(file); - // futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime))); + // 2024/5/13:使用多线程，并使用Future来获取结果，但是实际瓶颈可能是网络带宽而非计算 + // futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime,forecastHour))); ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour); finishedList.add(importResult); - // ImportResult importResult = importFromNcFile(meta, file, 0, forecastHour); if (importResult.isSuccess()) { forecastHours.add(String.valueOf(importResult.getForecastHour())); } } - } finally { - importing = false; - } // for (Future future : futures) { // ImportResult importResult = future.get(); - // forecastHours.add(String.valueOf(importResult.getForcastHour())); + // finishedList.add(importResult); + // forecastHours.add(String.valueOf(importResult.getForecastHour())); + // if (importResult.isSuccess()) { + // forecastHours.add(String.valueOf(importResult.getForecastHour())); + // } // } + + List failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList()); + if (!failedList.isEmpty()) { + log.warn("存在气象数据导入数据库失败，失败文件列表： {} ", failedList); + meta.addAttribute("status", "WRONG"); + } else { + meta.addAttribute("status", "DONE"); + } long end = System.currentTimeMillis(); log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours); meta.setForecastHours(forecastHours); @@ -143,15 +153,16 @@ public class GfsDataImport extends BaseTableOperation { putMeta(meta); gfsDataFetcher.lastGridDataSetMeta = meta; log.info("[tablestore]: 更新最新的数据元信息: {}", meta); + importing = false; return finishedList; } /** - * read data from netcdf file and write data to table store. - * + * 读取grib/nc文件，并进行相关计算，最后导入tablestore数据库 * @param meta 数据元信息 * @param file netcdf文件路径 * @param iTime 文件序号 + * @param forecastHour 预报时效（相对于起报时间的小时数） */ public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) { try (NetcdfFile ncFile = NetcdfFiles.open(file)) { @@ -191,7 +202,6 @@ public class GfsDataImport extends BaseTableOperation { new int[]{0, 0}, new int[]{xsize, ysize}); writer.writeGrid2D(variableName, iTime, 0, grid2D); } - } else { log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName); } @@ -218,8 +228,9 @@ public class GfsDataImport extends BaseTableOperation { } } + /** 计算风速风向，包括地面坐标和气压坐标，并导入数据库*/ private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception { - // TODO 2024/5/8: 风速风向需要保存到文件中 + // TODO 2024/5/8: 风速风向最好单独保存为一个文件，如果后续需要二维可视化，可以直接本地读取出图，减少网络请求 for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) { Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix); Variable vwnd = ncFile.findVariable("v-component_of_wind" + suffix); diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java deleted file mode 100644 index 80b317a..0000000 --- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.htfp.weather.griddata.operation; - -/** - * @Author : shiyi - * @Date : 2024/2/2 13:47 - * @Description : 获取数据 - */ -public interface IDataFetch { -} diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java index e05e5f2..fd9994f 100644 --- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java +++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java @@ -1,9 +1,8 @@ package com.htfp.weather.schedule; -import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.htfp.weather.download.FileInfo; -import com.htfp.weather.download.GfsDataConfig; -import com.htfp.weather.download.GfsDownloader; +import com.htfp.weather.download.gfs.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDownloader; import com.htfp.weather.griddata.operation.GfsDataFetcher; import com.htfp.weather.griddata.operation.GfsDataImport; import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult; @@ -23,7 +22,6 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -43,8 +41,6 @@ public class GridDataProcessor { @Resource GfsDownloader gfsDownloader; @Resource - GfsDataFetcher gfsDataFetcher; - @Resource GfsDataConfig gfsDataConfig; @Resource ErrorReportService errorReportService; @@ -54,8 +50,10 @@ public class GridDataProcessor { private final ExecutorService executorService = Executors.newFixedThreadPool(3); boolean test = false; + + /**每日北京时0,8,12,18点执行*/ @Scheduled(cron = "0 0 0,8,12,18 * * ?") - public void dailyDataProcess() throws Exception { + public void dailyDataProcess() { if (download()) { importToTableStore(); } diff --git a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java index da11e0a..7161c0f 100644 --- a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java +++ b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java @@ -9,7 +9,7 @@ import java.time.format.DateTimeFormatter; * @Description : 日期时间工具类 */ public class DateTimeUtils { - public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mmxxx"; + public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; public static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ofPattern(DEFAULT_PATTERN); /** * 将某时区的时间字符串更改为其他时区的时间字符串, diff --git a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java index cc84aaf..ca293d5 100644 --- a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java +++ b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java @@ -89,7 +89,7 @@ public class HttpClientUtils { } - public static boolean downloadFileByUrl(String sourceUrl, String localPath) throws IOException { + public static boolean downloadFileByUrl(String sourceUrl, String localPath) { File file = new File(localPath); Request request = new Request.Builder() .url(sourceUrl) @@ -102,8 +102,9 @@ public class HttpClientUtils { } else { return false; } + } catch (IOException e) { + return false; } - return true; } diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java index 6941629..8b46988 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java @@ -1,8 +1,8 @@ package com.htfp.weather.web.controller; import com.htfp.weather.download.FileInfo; -import com.htfp.weather.download.GfsDataConfig; -import com.htfp.weather.download.GfsDownloader; +import com.htfp.weather.download.gfs.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDownloader; import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.operation.GfsDataImport; import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult; diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java index 8f637ea..3ff8705 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java @@ -29,7 +29,7 @@ public class SurfaceWeatherController { @Resource(name = "hefeng") ISurfaceDataService surfaceDataService; - @PostMapping("/now") + @PostMapping("/querySurfaceNowWeather") public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception { double lat = position2D.getLatitude(); double lon = position2D.getLongitude(); @@ -41,8 +41,8 @@ public class SurfaceWeatherController { return Result.success(nowWeatherStatus); } - @PostMapping("/forecast") - public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception { + @PostMapping("/querySurfaceForecast") + public Result querySurfaceForecast(@Validated @RequestBody Position2D position2D) throws Exception { double lat = position2D.getLatitude(); double lon = position2D.getLongitude(); log.info("[data-server] 地面24小时预报结果查询 start: param={}", position2D); @@ -53,8 +53,8 @@ public class SurfaceWeatherController { return Result.success(forecastSeries); } - @PostMapping ("/warning") - public Result warning(@Validated @RequestBody Position2D position2D) throws Exception { + @PostMapping ("/queryWeatherWarning") + public Result queryWeatherWarning(@Validated @RequestBody Position2D position2D) throws Exception { double lat = position2D.getLatitude(); double lon = position2D.getLongitude(); log.info("[data-server] 地面气象预警信息查询 start: param={}", position2D); diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java index a951c15..beecc36 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java @@ -1,6 +1,5 @@ package com.htfp.weather.web.controller; -import com.htfp.weather.info.Constant; import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.ErrorCode; @@ -19,7 +18,7 @@ import java.util.List; /** * @Author : shiyi * @Date : 2024/5/8 15:15 - * @Description : + * @Description : 高空天气查询接口 */ @CrossOrigin @RestController @Slf4j @@ -61,8 +60,8 @@ public class UpperWeatherController { return Result.success(nowWeatherByMultiPoint); } - @PostMapping("/queryForecastTimeSeries24Hour") - public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) { + @PostMapping("/queryUpperForecast") + public Result queryUpperForecast(@Validated @RequestBody Position3D position3D) { double latitude = position3D.getLatitude(); double longitude = position3D.getLongitude(); int level = position3D.getLevel(); @@ -72,8 +71,8 @@ public class UpperWeatherController { return Result.success(forecastSeries); } - @PostMapping("/queryForecastTimeSeriesInMultiPoints") - public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) { + @PostMapping("/queryUpperForecastInMultiPoints") + public Result queryUpperForecastInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) { List latitudeList = new ArrayList<>(); List longitudeList = new ArrayList<>(); for (Position2D position : multiPointsRequest.getPointList()) { @@ -93,6 +92,8 @@ public class UpperWeatherController { return Result.success(nowWeatherByMultiPoint); } + + /**查询指定变量随气压的分布*/ @RequestMapping("/queryProfileByVariableAndPressure") public Result queryProfileByVariableAndPressure(@Validated @RequestBody ProfileRequest profileRequest) { OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime()); @@ -106,22 +107,34 @@ public class UpperWeatherController { return Result.success(profileResponse); } - @PostMapping("/plane") - public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) { + /**查询全部变量随气压的分布*/ + @RequestMapping("/queryProfileByPressure") + public Result queryProfileByAndPressure(@Validated @RequestBody ProfileRequest profileRequest) { + OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime()); + OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time); + double latitude = profileRequest.getLatitude(); + double longitude = profileRequest.getLongitude(); + log.info("[data-server] 高度廓线查询 start: param={}", profileRequest); + ProfileDataset profile = gfsDataService.getProfileByPressure(utcDateTime, latitude, longitude); + log.info("[data-server] 高度廓线查询 end"); + return Result.success(profile); + } + + @PostMapping("/queryPlaneGrid") + public Result queryPlaneGrid(@Validated @RequestBody PlaneRequest planeRequest) { planeRequest.valid(); String variableName = planeRequest.getVariableName(); OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime()); OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time); int level = planeRequest.getLevel(); - double minLat = planeRequest.getMinLat(); - double maxLat = planeRequest.getMaxLat(); - double minLon = planeRequest.getMinLon(); - double maxLon = planeRequest.getMaxLon(); + double minLat = planeRequest.getMinLatitude(); + double maxLat = planeRequest.getMaxLatitude(); + double minLon = planeRequest.getMinLongitude(); + double maxLon = planeRequest.getMaxLongitude(); log.info("[data-server] 平面网格数据查询 start: param={}", planeRequest); PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon); log.info("[data-server] 平面网格数据查询 end"); return Result.success(forecastSeries); } - } diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java index 0cac655..bdb3c84 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java @@ -1,6 +1,6 @@ package com.htfp.weather.web.pojo.request; -import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDataConfig; import lombok.Data; /** diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java index da44e2c..570e65e 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java @@ -1,8 +1,7 @@ package com.htfp.weather.web.pojo.request; -import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; -import com.htfp.weather.info.Constant; -import com.htfp.weather.info.GfsLevelsEnum; +import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum; +import com.htfp.weather.download.gfs.GfsLevelsEnum; import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.ErrorCode; import com.htfp.weather.web.valid.DateTimeStr; @@ -35,29 +34,29 @@ public class PlaneRequest { @NotNull @Min(value = 70, message = "经度最小值为70（东经180°）") @Max(value = 140, message = "经度最大值为140（东经140°）") - private Double minLon; + private Double minLongitude; @NotNull @Min(value = 70, message = "经度最小值为70（东经180°）") @Max(value = 140, message = "经度最大值为140（东经140°）") - private Double maxLon; + private Double maxLongitude; @NotNull @Min(value = 0, message = "纬度最小值为0（0°）") @Max(value = 55, message = "纬度最大值为55（北纬55°）") - private Double minLat; + private Double minLatitude; @NotNull @Min(value = 0, message = "纬度最小值为0（0°）") @Max(value = 55, message = "纬度最大值为55（北纬55°）") - private Double maxLat; + private Double maxLatitude; public void valid() { // TODO 2024/4/21: 校验 - if (minLon >= maxLon) { + if (minLongitude >= maxLongitude) { throw new AppException(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度"); } - if (minLat >= maxLat) { + if (minLatitude >= maxLatitude) { throw new AppException(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度"); } } diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java index c13e3cf..59d9ae1 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java @@ -19,7 +19,6 @@ public class ProfileRequest { @DateTimeStr(message = "时间格式错误") String time; - @NotNull(message = "变量名不能为空") String variableName; @NotNull(message = "纬度 (latitude) 不能为空") diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java new file mode 100644 index 0000000..0e8fafe --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java @@ -0,0 +1,42 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +import java.util.ArrayList; + +/** + * @Author : shiyi + * @Date : 2024/5/8 15:19 + * @Description : 高度廓线 + */ +@Data +public class ProfileDataset { + int[] pressureLevels; + // int[ pressureHeight; + double latitude; + double longitude; + + float[] temp; + float[] windSpeed; + float[] wind360; + // List windScale; + float[] humidity; + // float[] pressure; + float[] cloud; + float[] precip; + + + public ProfileDataset() {}; + public ProfileDataset(int size) { + this.pressureLevels = new int[size]; + // this.pressureHeight = pressureHeight; + temp = new float[size]; + windSpeed = new float[size]; + wind360 = new float[size]; + // windScale = new float[size; + humidity = new float[size]; + // pressure = new float[size; + cloud = new float[size]; + precip = new float[size]; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java index f9569d5..0d6dc7a 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java +++ b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java @@ -4,21 +4,19 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum; import com.aliyun.tablestore.grid.model.GridDataSet; import com.aliyun.tablestore.grid.model.GridDataSetMeta; import com.aliyun.tablestore.grid.model.grid.Grid4D; -import com.htfp.weather.griddata.common.GfsVariableHeightEnum; -import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; +import com.htfp.weather.download.gfs.GfsVariableHeightEnum; +import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum; import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.operation.GfsDataFetcher; -import com.htfp.weather.griddata.utils.ValueRange; +import com.htfp.weather.griddata.common.ValueRange; import com.htfp.weather.info.Constant; -import com.htfp.weather.info.GfsLevelsEnum; +import com.htfp.weather.download.gfs.GfsLevelsEnum; import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.ErrorCode; -import com.htfp.weather.web.pojo.response.NowWeatherStatus; -import com.htfp.weather.web.pojo.response.PlaneResponse; -import com.htfp.weather.web.pojo.response.ProfileResponse; -import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import com.htfp.weather.web.pojo.response.*; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import ucar.ma2.Array; @@ -30,7 +28,6 @@ import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.*; /** @@ -201,9 +198,23 @@ public class GfsDataServiceImpl implements IDataService { } catch (Exception e) { throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage()); } - + } + public ProfileDataset getProfileByPressure(OffsetDateTime targetTime, double latitude, double longitude) { + try { + List variableNames = getVariableNamesInDatabase(GfsLevelsEnum.UPPER.getCode()); + GridDataSet gridDataSet = gfsDataFetcher.getProfileByPressure(variableNames, targetTime, latitude, longitude); + ProfileDataset profile = buildProfileDataset(gridDataSet, GfsLevelsEnum.UPPER); + profile.setLatitude(latitude); + profile.setLongitude(longitude); + return profile; + } catch (AppException e) { + throw e; + } catch (Exception e) { + throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage()); + } } + /** * 获取指定变量、时间、高度的二维数组数据 * @@ -304,16 +315,7 @@ public class GfsDataServiceImpl implements IDataService { // 利用反射构建数据集 Class aClass = result.getClass(); for (Map.Entry entry : gridDataSet.getVariables().entrySet()) { - String variableNameInFile = entry.getKey(); - String variableNameInApi; - if (GfsLevelsEnum.SURFACE.equals(levelFlag)) { - variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile); - } else { - variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile); - } - if (variableNameInApi == null) { - throw new AppException(ErrorCode.NO_SUCH_VARIABLE); - } + String variableNameInApi = getVariableNameInApi(levelFlag, entry); Grid4D grid4D = entry.getValue(); int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引 int[] shape = grid4D.getShape(); @@ -381,16 +383,7 @@ public class GfsDataServiceImpl implements IDataService { // 利用反射构建数据集 Class aClass = result.getClass(); for (Map.Entry entry : gridDataSet.getVariables().entrySet()) { - String variableNameInFile = entry.getKey(); - String variableNameInApi; - if (GfsLevelsEnum.SURFACE.equals(levelFlag)) { - variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile); - } else { - variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile); - } - if (variableNameInApi == null) { - throw new AppException(ErrorCode.NO_SUCH_VARIABLE); - } + String variableNameInApi = getVariableNameInApi(levelFlag, entry); Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape()); int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引 try { @@ -410,6 +403,48 @@ public class GfsDataServiceImpl implements IDataService { return result; } + private ProfileDataset buildProfileDataset(GridDataSet gridDataSet, GfsLevelsEnum levelFlag) { + ProfileDataset result = new ProfileDataset(); + // 利用反射构建数据集 + Class aClass = result.getClass(); + for (Map.Entry entry : gridDataSet.getVariables().entrySet()) { + try { + String variableNameInApi = getVariableNameInApi(levelFlag, entry); + Grid4D grid4D = entry.getValue(); + int[] shape = grid4D.getShape(); + int zSize = shape[1]; + String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1); + Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType()); + // TODO 2024/6/20: 这里的toArray操作在数据量较大的时候会比较吃内存和cpu + float[] values = (float[]) grid4D.toArray().section( + new int[]{0, 0, 0, 0}, new int[]{1, zSize, 1, 1} + ).copyTo1DJavaArray(); + method.invoke(result, values); + } catch (ReflectiveOperationException e) { + throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e); + } catch (Exception e) { + e.printStackTrace(); + throw new AppException(ErrorCode.RESPONSE_DATA_BUILD_ERROR); + } + } + result.setPressureLevels(tableConfig.getPressureList()); + return result; + } + + @NotNull + private static String getVariableNameInApi(GfsLevelsEnum levelFlag, Map.Entry variable) { + String variableNameInFile = variable.getKey(); + String variableNameInApi; + if (GfsLevelsEnum.SURFACE.equals(levelFlag)) { + variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile); + } else { + variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile); + } + if (variableNameInApi == null) { + throw new AppException(ErrorCode.NO_SUCH_VARIABLE); + } + return variableNameInApi; + } private Map getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) { long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName()); diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java index ce73e73..4e7c2b7 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java @@ -15,6 +15,8 @@ import com.htfp.weather.web.pojo.response.TimeSeriesDataset; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -141,7 +143,9 @@ public class CaiYunServiceImpl implements ISurfaceDataService { try { TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(24); for (int i = 0; i < 24; i++) { - timeSeriesDataset.getTime().add(hourly.getTemperature().get(i).getDatetime()); + timeSeriesDataset.getTime().add( + OffsetDateTime.parse(hourly.getTemperature().get(i).getDatetime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); timeSeriesDataset.getTemp()[i] = hourly.getTemperature().get(i).getValue(); timeSeriesDataset.getWindSpeed()[i] = (float) MeteoUtils.kmPerHour2mPerSecond(hourly.getWind().get(i).getSpeed()); timeSeriesDataset.getWind360()[i] = hourly.getWind().get(i).getDirection(); diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java index 4ac416f..7e2eb8e 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; import java.security.MessageDigest; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import static com.htfp.weather.utils.HttpClientUtils.*; @@ -194,7 +196,9 @@ public class HeFengServiceImpl implements ISurfaceDataService { List surfaceWeatherWarnings = new ArrayList<>(); for (HeFengWarning heFengWarning : heFengWarnings) { SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning(); - surfaceWeatherWarning.setPubTime(heFengWarning.getPubTime()); + surfaceWeatherWarning.setPubTime( + OffsetDateTime.parse(heFengWarning.getPubTime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); // surfaceWeatherWarning.setStartTime(heFengWarning.getStartTime()); surfaceWeatherWarning.setTitle(heFengWarning.getTitle()); surfaceWeatherWarning.setText(heFengWarning.getText()); @@ -213,13 +217,12 @@ public class HeFengServiceImpl implements ISurfaceDataService { * @return */ private BaseHeFengData processHeFengData(BaseHeFengData data) { - String format = Constant.API_TIME_STRING; if (data instanceof HeFengForecastHour) { - String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format); - ((HeFengForecastHour) data).setFxTime(fxTime); + String fxTime = ((HeFengForecastHour) data).getFxTime(); + ((HeFengForecastHour) data).setFxTime(OffsetDateTime.parse(fxTime).format(Constant.API_TIME_FORMATTER)); } else if (data instanceof HeFengNow) { - String obsTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengNow) data).getObsTime(), format); - ((HeFengNow) data).setObsTime(obsTime); + String obsTime = ((HeFengNow) data).getObsTime(); + ((HeFengNow) data).setObsTime(OffsetDateTime.parse(obsTime).format(Constant.API_TIME_FORMATTER)); } float windSpeed = (float) MeteoUtils.kmPerHour2mPerSecond(data.getWindSpeed()); data.setWindSpeed(windSpeed); diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java index 41957c7..83ab309 100644 --- a/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java +++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java @@ -1,11 +1,10 @@ package com.htfp.weather.download; +import com.htfp.weather.download.gfs.GfsDataConfig; import org.junit.jupiter.api.Test; import java.io.IOException; -import static org.junit.jupiter.api.Assertions.*; - /** * @Author : shiyi * @Date : 2024/4/21 16:28 diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java index a1e0b2b..8eba0cf 100644 --- a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java +++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java @@ -1,11 +1,11 @@ package com.htfp.weather.download; +import com.htfp.weather.download.gfs.GfsDataConfig; +import com.htfp.weather.download.gfs.GfsDownloader; import org.junit.jupiter.api.Test; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - /** * @Author : shiyi * @Date : 2024/4/25 16:15 diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java index 71d1b00..1cd50fb 100644 --- a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java +++ b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java @@ -1,6 +1,6 @@ package com.htfp.weather.griddata.operation; -import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; +import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @@ -31,7 +31,6 @@ class GfsDataFetcherTest { @Test void getProfile() throws Exception { log.info("获取廓线"); - gfsDataFetcher.getProfile(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Temp"), 9,0, 0); } // @Test diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java deleted file mode 100644 index a40da91..0000000 --- a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.htfp.weather.griddata.utils; - -import com.htfp.weather.utils.DateTimeUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFiles; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @Author : shiyi - * @Date : 2024/3/4 10:21 - * @Description : - */ -class GfsUtilsTest { - - @Test - void getRefTime() throws IOException { - String filename = "D:\\HTFP\\weather\\GFSData\\UTC-20240607.00\\BJT-20240607.17-from-UTC-20240607.00+9.grib2"; - NetcdfFile ncFile = NetcdfFiles.open(filename); - String refTime = GfsUtils.getRefTime(ncFile); - String s = filename.split("\\+")[1]; - Integer.parseInt(s.split(".grib2")[0]); - // assertEquals("2023-09-05T00:00+00:00", refTime); - // Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN)); - } -} \ No newline at end of file