From 2ec081e80f79904724afc07c90f18fc788cd23f5 Mon Sep 17 00:00:00 2001
From: shiyi <shi_yee@outlook.com>
Date: Mon, 1 Jul 2024 17:55:23 +0800
Subject: [PATCH] =?UTF-8?q?package=E9=83=A8=E5=88=86=E9=87=8D=E6=96=B0?=
=?UTF-8?q?=E7=BB=84=E7=BB=87=EF=BC=9B=E5=86=97=E4=BD=99=E4=BB=A3=E7=A0=81?=
=?UTF-8?q?=E5=88=A0=E9=99=A4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../java/com/htfp/weather/config/Config.java | 18 ----
.../weather/download/BaseDataDownloader.java | 13 +--
.../download/{ => gfs}/GfsDataConfig.java | 4 +-
.../gfs}/GfsDownloadVariableEnum.java | 2 +-
.../download/{ => gfs}/GfsDownloader.java | 27 ++----
.../{info => download/gfs}/GfsLevelsEnum.java | 3 +-
.../gfs}/GfsVariableHeightEnum.java | 4 +-
.../gfs}/GfsVariableIsobaricEnum.java | 4 +-
.../weather/griddata/common/TableConfig.java | 2 +-
.../griddata/common/TableConfigStatic.java | 2 +-
.../{utils => common}/ValueRange.java | 4 +-
.../griddata/operation/GfsDataFetcher.java | 18 +++-
.../griddata/operation/GfsDataImport.java | 51 ++++++----
.../griddata/operation/IDataFetch.java | 9 --
.../weather/schedule/GridDataProcessor.java | 12 +--
.../com/htfp/weather/utils/DateTimeUtils.java | 2 +-
.../htfp/weather/utils/HttpClientUtils.java | 5 +-
.../web/controller/ConfigController.java | 4 +-
.../controller/SurfaceWeatherController.java | 10 +-
.../controller/UpperWeatherController.java | 39 +++++---
.../web/pojo/request/GfsConfigUpdate.java | 2 +-
.../web/pojo/request/PlaneRequest.java | 17 ++--
.../web/pojo/request/ProfileRequest.java | 1 -
.../web/pojo/response/ProfileDataset.java | 42 ++++++++
.../web/service/GfsDataServiceImpl.java | 95 +++++++++++++------
.../service/surfaceapi/CaiYunServiceImpl.java | 6 +-
.../service/surfaceapi/HeFengServiceImpl.java | 15 +--
.../weather/download/GfsDataConfigTest.java | 3 +-
.../weather/download/GfsDownloaderTest.java | 4 +-
.../operation/GfsDataFetcherTest.java | 3 +-
.../weather/griddata/utils/GfsUtilsTest.java | 30 ------
31 files changed, 249 insertions(+), 202 deletions(-)
delete mode 100644 weather-service/src/main/java/com/htfp/weather/config/Config.java
rename weather-service/src/main/java/com/htfp/weather/download/{ => gfs}/GfsDataConfig.java (97%)
rename weather-service/src/main/java/com/htfp/weather/{info => download/gfs}/GfsDownloadVariableEnum.java (96%)
rename weather-service/src/main/java/com/htfp/weather/download/{ => gfs}/GfsDownloader.java (93%)
rename weather-service/src/main/java/com/htfp/weather/{info => download/gfs}/GfsLevelsEnum.java (95%)
rename weather-service/src/main/java/com/htfp/weather/{griddata/common => download/gfs}/GfsVariableHeightEnum.java (94%)
rename weather-service/src/main/java/com/htfp/weather/{griddata/common => download/gfs}/GfsVariableIsobaricEnum.java (94%)
rename weather-service/src/main/java/com/htfp/weather/griddata/{utils => common}/ValueRange.java (85%)
delete mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java
create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java
delete mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java
diff --git a/weather-service/src/main/java/com/htfp/weather/config/Config.java b/weather-service/src/main/java/com/htfp/weather/config/Config.java
deleted file mode 100644
index 33979cc..0000000
--- a/weather-service/src/main/java/com/htfp/weather/config/Config.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package com.htfp.weather.config;
-
-import com.htfp.weather.griddata.common.TableConfig;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.Resource;
-
-/**
- * @Author : shiyi
- * @Date : 2024/1/22 19:02
- * @Description :
- */
-@Component("config")
-public class Config {
- @Resource
- TableConfig tableConfig;
-
-}
diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
index 73fb052..633eef9 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
@@ -8,8 +8,9 @@ import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.util.List;
-/**
- * @author shiyi
+/** @Author : shiyi
+ * @Date : 2024/2/28 13:53
+ * @Description : GFS数据下载类
*/
@Data
@ToString
@@ -36,16 +37,10 @@ public abstract class BaseDataDownloader {
/**
* 下载所有目标文件
- *
- * @param filesInfo
+ * @param fileInfoList
* @return
*/
public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
-
- public BaseDataDownloader() {
-
- }
-
}
diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java
similarity index 97%
rename from weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java
index 5b9a964..2d21a20 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDataConfig.java
@@ -1,8 +1,6 @@
-package com.htfp.weather.download;
+package com.htfp.weather.download.gfs;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.htfp.weather.info.GfsDownloadVariableEnum;
-import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.JSONUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java
similarity index 96%
rename from weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java
index 954d5a2..b546ba6 100644
--- a/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloadVariableEnum.java
@@ -1,4 +1,4 @@
-package com.htfp.weather.info;
+package com.htfp.weather.download.gfs;
/**
* @Author : shiyi
diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java
similarity index 93%
rename from weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java
index 1304f76..7eb02a0 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsDownloader.java
@@ -1,10 +1,11 @@
-package com.htfp.weather.download;
+package com.htfp.weather.download.gfs;
+import com.htfp.weather.download.BaseDataDownloader;
+import com.htfp.weather.download.FileInfo;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.HttpClientUtils;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
@@ -13,12 +14,7 @@ import ucar.nc2.NetcdfFiles;
import javax.annotation.Resource;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.net.ConnectException;
-import java.net.URL;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
@@ -30,6 +26,7 @@ import java.util.concurrent.Future;
/**
* @author shiyi
+ * @
*/
@Slf4j
@Component("gfsDownloader")
@@ -42,7 +39,6 @@ public class GfsDownloader extends BaseDataDownloader {
private OffsetDateTime realUpdateDateTime;
public GfsDownloader(GfsDataConfig gfsDataConfig) {
- super();
this.gfsDataConfig = gfsDataConfig;
}
@@ -58,8 +54,8 @@ public class GfsDownloader extends BaseDataDownloader {
realUpdateDateTime = getAvailableUpdateDateTime(utcStartTime);
this.setRefTime(realUpdateDateTime);
this.setRefTimeStr(realUpdateDateTime.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)));
- // 起报点和实际请求时间的差异
+ // 起报点和实际请求时间的差异
this.hourDiff = (int) Duration.between(realUpdateDateTime, utcStartTime).toHours();
log.info("[GFS Download] 起始为当前本地时间：{}", this.getStartTime());
log.info("[GFS Download] GFS最近更新时间UTC：{}，本地时间: {}", realUpdateDateTime, DateTimeUtils.getLocalZoneDateTime(realUpdateDateTime));
@@ -69,24 +65,20 @@ public class GfsDownloader extends BaseDataDownloader {
public FileInfo download(FileInfo fileInfo) {
return download0(fileInfo, 2);
}
- private FileInfo download0(FileInfo fileInfo, int retryNum) {
+ private FileInfo download0(FileInfo fileInfo, int retryNum) {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
File fileOut = new File(destDir, fileInfo.getFileName());
log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
try {
- // TODO 2024/5/24: 增加重试机制
- // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
- // throw new RuntimeException();
- // }
- // FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
- // FIXME 2024/6/8: 如果连接是https，jar包下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
+ // DONE: 改用okhttp FIXME 2024/6/8: 如果连接是https，jar包启动下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
HttpClientUtils.downloadFileByUrl(url, fileOut.getPath());
log.info("[GFS Download] 文件下载成功: {}", fileOut);
fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
+ // DONE 2024/5/24: 文件服务器在外网不稳定，增加重试机制
fileInfo.setDownloadSuccess(false);
if (retryNum > 0) {
log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut);
@@ -99,6 +91,7 @@ public class GfsDownloader extends BaseDataDownloader {
return fileInfo;
}
+ /** 简单校验验证文件完整性，并生成索引文件*/
private boolean fileValid(String file) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
ncFile.getLocation();
@@ -274,7 +267,7 @@ public class GfsDownloader extends BaseDataDownloader {
for (String variable : variables) {
stringJoiner.add(String.format("var_%s=on", variable));
}
- stringJoiner.add("lev_surface=on"); // gust需要
+ stringJoiner.add("lev_surface=on"); // gust, prate需要
return stringJoiner.toString();
}
diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java
similarity index 95%
rename from weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java
index bdb7243..66b4c6e 100644
--- a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsLevelsEnum.java
@@ -1,4 +1,4 @@
-package com.htfp.weather.info;
+package com.htfp.weather.download.gfs;
/**
* @Author : shiyi
@@ -8,6 +8,7 @@ package com.htfp.weather.info;
public enum GfsLevelsEnum {
//
SURFACE(9999, "地面"),
+ UPPER(0, "高空"),
PRES_1000hPa(1000, "1000 hPa"),
PRES_975hPa(975, "975 hPa"),
PRES_950hPa(950, "950 hPa"),
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java
similarity index 94%
rename from weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java
index d769374..75feacc 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableHeightEnum.java
@@ -1,4 +1,4 @@
-package com.htfp.weather.griddata.common;
+package com.htfp.weather.download.gfs;
import java.util.ArrayList;
import java.util.List;
@@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
- * @Description : 文件变量名和表中变量名的映射
+ * @Description : 文件高度坐标变量名和表中变量名的映射
*/
public enum GfsVariableHeightEnum {
//
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java
similarity index 94%
rename from weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java
rename to weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java
index defb295..6051dec 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/gfs/GfsVariableIsobaricEnum.java
@@ -1,4 +1,4 @@
-package com.htfp.weather.griddata.common;
+package com.htfp.weather.download.gfs;
import java.util.ArrayList;
import java.util.List;
@@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
- * @Description : 文件变量名和表中变量名的映射
+ * @Description : 文件气压坐标变量名和表中变量名的映射
*/
public enum GfsVariableIsobaricEnum {
//
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java
index 4f9e376..e5e7991 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java
+++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java
@@ -2,7 +2,7 @@ package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.htfp.weather.download.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java
index 607e605..bcf7b63 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java
+++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigStatic.java
@@ -1,7 +1,7 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
-import com.htfp.weather.download.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDataConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java
similarity index 85%
rename from weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java
rename to weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java
index aa8ae90..b0c05bb 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/utils/ValueRange.java
+++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/ValueRange.java
@@ -1,9 +1,11 @@
-package com.htfp.weather.griddata.utils;
+package com.htfp.weather.griddata.common;
import lombok.Getter;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
/**
* @Author : shiyi
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java
index c9a30b1..9c4a5f7 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java
+++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java
@@ -7,8 +7,8 @@ import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Range;
import com.htfp.weather.griddata.common.TableConfig;
-import com.htfp.weather.griddata.utils.ValueRange;
-import com.htfp.weather.info.GfsLevelsEnum;
+import com.htfp.weather.griddata.common.ValueRange;
+import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
@@ -154,7 +154,19 @@ public class GfsDataFetcher extends BaseTableOperation {
Array array = gridDataSet.getVariable(variableName).toArray();
return array;
}
-
+ public GridDataSet getProfileByPressure(List<String> variableNameList, OffsetDateTime targetTime, double latitude, double longitude) throws Exception {
+ lastGridDataSetMeta = getLastGridDataSetMeta();
+ GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(lastGridDataSetMeta);
+ int iTime = getTargetTimeIndex(targetTime);
+ int iLat= getLatitudeIndex(latitude);
+ int iLon = getLongitudeIndex(longitude);
+ int[] origin = new int[]{iTime, 0, iLat, iLon};
+ int[] shape = new int[]{1,tableConfig.levSize, 1, 1};
+ // TODO 2024/6/17:
+ fetcher.setVariablesToGet(variableNameList);
+ fetcher.setOriginShape(origin, shape);
+ return fetcher.fetch();
+ }
/**
* 获取指定变量在某经纬高处的预报序列
* @param dataSetId 数据集ID
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java
index 4ecfd8a..a640cdc 100644
--- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java
+++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java
@@ -5,8 +5,9 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
-import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
-import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
+import com.htfp.weather.download.FileInfo;
+import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
+import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
@@ -32,6 +33,8 @@ import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
/**
* @author shi_y
@@ -85,57 +88,64 @@ public class GfsDataImport extends BaseTableOperation {
* @throws Exception
*/
public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception {
- meta.addAttribute("status", "DONE");
tableStoreGrid.updateDataSetMeta(meta);
return meta;
}
public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception {
- meta.addAttribute("status", "DONE");
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
+ /** 导入指定起报时间的所有文件*/
public List<ImportResult> importData(OffsetDateTime refTime) throws Exception {
+ importing = true;
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES);
}
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
+ // datasetId和起报时间绑定
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
- List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
-
- // done 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长
- // done 2024/5/13: 优化方案，使用多线程，并使用Future来获取结果，但是必须保证每个时刻都成功导入
+ List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
+ // todo 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长
List<ImportResult> finishedList;
- try {
+ // List<Future<ImportResult>> futures = new ArrayList<>();
finishedList = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
- // for(int i = 0; i < 200; i++) {
String file = fileList.get(i);
int iTime = i;
int forecastHour = getForecastHourFromFilename(file);
- // futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
+ // 2024/5/13:使用多线程，并使用Future来获取结果，但是实际瓶颈可能是网络带宽而非计算
+ // futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime,forecastHour)));
ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour);
finishedList.add(importResult);
- // ImportResult importResult = importFromNcFile(meta, file, 0, forecastHour);
if (importResult.isSuccess()) {
forecastHours.add(String.valueOf(importResult.getForecastHour()));
}
}
- } finally {
- importing = false;
- }
// for (Future<ImportResult> future : futures) {
// ImportResult importResult = future.get();
- // forecastHours.add(String.valueOf(importResult.getForcastHour()));
+ // finishedList.add(importResult);
+ // forecastHours.add(String.valueOf(importResult.getForecastHour()));
+ // if (importResult.isSuccess()) {
+ // forecastHours.add(String.valueOf(importResult.getForecastHour()));
+ // }
// }
+
+ List<ImportResult> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
+ if (!failedList.isEmpty()) {
+ log.warn("存在气象数据导入数据库失败，失败文件列表： {} ", failedList);
+ meta.addAttribute("status", "WRONG");
+ } else {
+ meta.addAttribute("status", "DONE");
+ }
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
@@ -143,15 +153,16 @@ public class GfsDataImport extends BaseTableOperation {
putMeta(meta);
gfsDataFetcher.lastGridDataSetMeta = meta;
log.info("[tablestore]: 更新最新的数据元信息: {}", meta);
+ importing = false;
return finishedList;
}
/**
- * read data from netcdf file and write data to table store.
- *
+ * 读取grib/nc文件，并进行相关计算，最后导入tablestore数据库
* @param meta 数据元信息
* @param file netcdf文件路径
* @param iTime 文件序号
+ * @param forecastHour 预报时效（相对于起报时间的小时数）
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
@@ -191,7 +202,6 @@ public class GfsDataImport extends BaseTableOperation {
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, 0, grid2D);
}
-
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
@@ -218,8 +228,9 @@ public class GfsDataImport extends BaseTableOperation {
}
}
+ /** 计算风速风向，包括地面坐标和气压坐标，并导入数据库*/
private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
- // TODO 2024/5/8: 风速风向需要保存到文件中
+ // TODO 2024/5/8: 风速风向最好单独保存为一个文件，如果后续需要二维可视化，可以直接本地读取出图，减少网络请求
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
Variable vwnd = ncFile.findVariable("v-component_of_wind" + suffix);
diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java
deleted file mode 100644
index 80b317a..0000000
--- a/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.htfp.weather.griddata.operation;
-
-/**
- * @Author : shiyi
- * @Date : 2024/2/2 13:47
- * @Description : 获取数据
- */
-public interface IDataFetch {
-}
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
index e05e5f2..fd9994f 100644
--- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
@@ -1,9 +1,8 @@
package com.htfp.weather.schedule;
-import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.htfp.weather.download.FileInfo;
-import com.htfp.weather.download.GfsDataConfig;
-import com.htfp.weather.download.GfsDownloader;
+import com.htfp.weather.download.gfs.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
@@ -23,7 +22,6 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -43,8 +41,6 @@ public class GridDataProcessor {
@Resource
GfsDownloader gfsDownloader;
@Resource
- GfsDataFetcher gfsDataFetcher;
- @Resource
GfsDataConfig gfsDataConfig;
@Resource
ErrorReportService errorReportService;
@@ -54,8 +50,10 @@ public class GridDataProcessor {
private final ExecutorService executorService = Executors.newFixedThreadPool(3);
boolean test = false;
+
+ /**每日北京时0,8,12,18点执行*/
@Scheduled(cron = "0 0 0,8,12,18 * * ?")
- public void dailyDataProcess() throws Exception {
+ public void dailyDataProcess() {
if (download()) {
importToTableStore();
}
diff --git a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java
index da11e0a..7161c0f 100644
--- a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java
+++ b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java
@@ -9,7 +9,7 @@ import java.time.format.DateTimeFormatter;
* @Description : 日期时间工具类
*/
public class DateTimeUtils {
- public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mmxxx";
+ public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx";
public static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ofPattern(DEFAULT_PATTERN);
/**
* 将某时区的时间字符串更改为其他时区的时间字符串,
diff --git a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java
index cc84aaf..ca293d5 100644
--- a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java
+++ b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java
@@ -89,7 +89,7 @@ public class HttpClientUtils {
}
- public static boolean downloadFileByUrl(String sourceUrl, String localPath) throws IOException {
+ public static boolean downloadFileByUrl(String sourceUrl, String localPath) {
File file = new File(localPath);
Request request = new Request.Builder()
.url(sourceUrl)
@@ -102,8 +102,9 @@ public class HttpClientUtils {
} else {
return false;
}
+ } catch (IOException e) {
+ return false;
}
-
return true;
}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
index 6941629..8b46988 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
@@ -1,8 +1,8 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.download.FileInfo;
-import com.htfp.weather.download.GfsDataConfig;
-import com.htfp.weather.download.GfsDownloader;
+import com.htfp.weather.download.gfs.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java
index 8f637ea..3ff8705 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java
@@ -29,7 +29,7 @@ public class SurfaceWeatherController {
@Resource(name = "hefeng")
ISurfaceDataService surfaceDataService;
- @PostMapping("/now")
+ @PostMapping("/querySurfaceNowWeather")
public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
@@ -41,8 +41,8 @@ public class SurfaceWeatherController {
return Result.success(nowWeatherStatus);
}
- @PostMapping("/forecast")
- public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception {
+ @PostMapping("/querySurfaceForecast")
+ public Result querySurfaceForecast(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
log.info("[data-server] 地面24小时预报结果查询 start: param={}", position2D);
@@ -53,8 +53,8 @@ public class SurfaceWeatherController {
return Result.success(forecastSeries);
}
- @PostMapping ("/warning")
- public Result warning(@Validated @RequestBody Position2D position2D) throws Exception {
+ @PostMapping ("/queryWeatherWarning")
+ public Result queryWeatherWarning(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
log.info("[data-server] 地面气象预警信息查询 start: param={}", position2D);
diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java
index a951c15..beecc36 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java
@@ -1,6 +1,5 @@
package com.htfp.weather.web.controller;
-import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
@@ -19,7 +18,7 @@ import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:15
- * @Description :
+ * @Description : 高空天气查询接口
*/
@CrossOrigin
@RestController @Slf4j
@@ -61,8 +60,8 @@ public class UpperWeatherController {
return Result.success(nowWeatherByMultiPoint);
}
- @PostMapping("/queryForecastTimeSeries24Hour")
- public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) {
+ @PostMapping("/queryUpperForecast")
+ public Result queryUpperForecast(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
@@ -72,8 +71,8 @@ public class UpperWeatherController {
return Result.success(forecastSeries);
}
- @PostMapping("/queryForecastTimeSeriesInMultiPoints")
- public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
+ @PostMapping("/queryUpperForecastInMultiPoints")
+ public Result queryUpperForecastInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
@@ -93,6 +92,8 @@ public class UpperWeatherController {
return Result.success(nowWeatherByMultiPoint);
}
+
+ /**查询指定变量随气压的分布*/
@RequestMapping("/queryProfileByVariableAndPressure")
public Result queryProfileByVariableAndPressure(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
@@ -106,22 +107,34 @@ public class UpperWeatherController {
return Result.success(profileResponse);
}
- @PostMapping("/plane")
- public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) {
+ /**查询全部变量随气压的分布*/
+ @RequestMapping("/queryProfileByPressure")
+ public Result queryProfileByAndPressure(@Validated @RequestBody ProfileRequest profileRequest) {
+ OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
+ OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
+ double latitude = profileRequest.getLatitude();
+ double longitude = profileRequest.getLongitude();
+ log.info("[data-server] 高度廓线查询 start: param={}", profileRequest);
+ ProfileDataset profile = gfsDataService.getProfileByPressure(utcDateTime, latitude, longitude);
+ log.info("[data-server] 高度廓线查询 end");
+ return Result.success(profile);
+ }
+
+ @PostMapping("/queryPlaneGrid")
+ public Result queryPlaneGrid(@Validated @RequestBody PlaneRequest planeRequest) {
planeRequest.valid();
String variableName = planeRequest.getVariableName();
OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
int level = planeRequest.getLevel();
- double minLat = planeRequest.getMinLat();
- double maxLat = planeRequest.getMaxLat();
- double minLon = planeRequest.getMinLon();
- double maxLon = planeRequest.getMaxLon();
+ double minLat = planeRequest.getMinLatitude();
+ double maxLat = planeRequest.getMaxLatitude();
+ double minLon = planeRequest.getMinLongitude();
+ double maxLon = planeRequest.getMaxLongitude();
log.info("[data-server] 平面网格数据查询 start: param={}", planeRequest);
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
log.info("[data-server] 平面网格数据查询 end");
return Result.success(forecastSeries);
}
-
}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java
index 0cac655..bdb3c84 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/GfsConfigUpdate.java
@@ -1,6 +1,6 @@
package com.htfp.weather.web.pojo.request;
-import com.htfp.weather.download.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDataConfig;
import lombok.Data;
/**
diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java
index da44e2c..570e65e 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java
@@ -1,8 +1,7 @@
package com.htfp.weather.web.pojo.request;
-import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
-import com.htfp.weather.info.Constant;
-import com.htfp.weather.info.GfsLevelsEnum;
+import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
+import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.valid.DateTimeStr;
@@ -35,29 +34,29 @@ public class PlaneRequest {
@NotNull
@Min(value = 70, message = "经度最小值为70（东经180°）")
@Max(value = 140, message = "经度最大值为140（东经140°）")
- private Double minLon;
+ private Double minLongitude;
@NotNull
@Min(value = 70, message = "经度最小值为70（东经180°）")
@Max(value = 140, message = "经度最大值为140（东经140°）")
- private Double maxLon;
+ private Double maxLongitude;
@NotNull
@Min(value = 0, message = "纬度最小值为0（0°）")
@Max(value = 55, message = "纬度最大值为55（北纬55°）")
- private Double minLat;
+ private Double minLatitude;
@NotNull
@Min(value = 0, message = "纬度最小值为0（0°）")
@Max(value = 55, message = "纬度最大值为55（北纬55°）")
- private Double maxLat;
+ private Double maxLatitude;
public void valid() {
// TODO 2024/4/21: 校验
- if (minLon >= maxLon) {
+ if (minLongitude >= maxLongitude) {
throw new AppException(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度");
}
- if (minLat >= maxLat) {
+ if (minLatitude >= maxLatitude) {
throw new AppException(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度");
}
}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java
index c13e3cf..59d9ae1 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java
@@ -19,7 +19,6 @@ public class ProfileRequest {
@DateTimeStr(message = "时间格式错误")
String time;
- @NotNull(message = "变量名不能为空")
String variableName;
@NotNull(message = "纬度 (latitude) 不能为空")
diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java
new file mode 100644
index 0000000..0e8fafe
--- /dev/null
+++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileDataset.java
@@ -0,0 +1,42 @@
+package com.htfp.weather.web.pojo.response;
+
+import lombok.Data;
+
+import java.util.ArrayList;
+
+/**
+ * @Author : shiyi
+ * @Date : 2024/5/8 15:19
+ * @Description : 高度廓线
+ */
+@Data
+public class ProfileDataset {
+ int[] pressureLevels;
+ // int[ pressureHeight;
+ double latitude;
+ double longitude;
+
+ float[] temp;
+ float[] windSpeed;
+ float[] wind360;
+ // List<String> windScale;
+ float[] humidity;
+ // float[] pressure;
+ float[] cloud;
+ float[] precip;
+
+
+ public ProfileDataset() {};
+ public ProfileDataset(int size) {
+ this.pressureLevels = new int[size];
+ // this.pressureHeight = pressureHeight;
+ temp = new float[size];
+ windSpeed = new float[size];
+ wind360 = new float[size];
+ // windScale = new float[size;
+ humidity = new float[size];
+ // pressure = new float[size;
+ cloud = new float[size];
+ precip = new float[size];
+ }
+}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java
index f9569d5..0d6dc7a 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java
@@ -4,21 +4,19 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
-import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
-import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
+import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
+import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
-import com.htfp.weather.griddata.utils.ValueRange;
+import com.htfp.weather.griddata.common.ValueRange;
import com.htfp.weather.info.Constant;
-import com.htfp.weather.info.GfsLevelsEnum;
+import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
-import com.htfp.weather.web.pojo.response.NowWeatherStatus;
-import com.htfp.weather.web.pojo.response.PlaneResponse;
-import com.htfp.weather.web.pojo.response.ProfileResponse;
-import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
+import com.htfp.weather.web.pojo.response.*;
import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import ucar.ma2.Array;
@@ -30,7 +28,6 @@ import java.time.Duration;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
import java.util.*;
/**
@@ -201,9 +198,23 @@ public class GfsDataServiceImpl implements IDataService {
} catch (Exception e) {
throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage());
}
-
+ }
+ public ProfileDataset getProfileByPressure(OffsetDateTime targetTime, double latitude, double longitude) {
+ try {
+ List<String> variableNames = getVariableNamesInDatabase(GfsLevelsEnum.UPPER.getCode());
+ GridDataSet gridDataSet = gfsDataFetcher.getProfileByPressure(variableNames, targetTime, latitude, longitude);
+ ProfileDataset profile = buildProfileDataset(gridDataSet, GfsLevelsEnum.UPPER);
+ profile.setLatitude(latitude);
+ profile.setLongitude(longitude);
+ return profile;
+ } catch (AppException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage());
+ }
}
+
/**
* 获取指定变量、时间、高度的二维数组数据
*
@@ -304,16 +315,7 @@ public class GfsDataServiceImpl implements IDataService {
// 利用反射构建数据集
Class<? extends TimeSeriesDataset> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
- String variableNameInFile = entry.getKey();
- String variableNameInApi;
- if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
- variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
- } else {
- variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
- }
- if (variableNameInApi == null) {
- throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
- }
+ String variableNameInApi = getVariableNameInApi(levelFlag, entry);
Grid4D grid4D = entry.getValue();
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
int[] shape = grid4D.getShape();
@@ -381,16 +383,7 @@ public class GfsDataServiceImpl implements IDataService {
// 利用反射构建数据集
Class<? extends NowWeatherStatus> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
- String variableNameInFile = entry.getKey();
- String variableNameInApi;
- if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
- variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
- } else {
- variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
- }
- if (variableNameInApi == null) {
- throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
- }
+ String variableNameInApi = getVariableNameInApi(levelFlag, entry);
Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape());
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
try {
@@ -410,6 +403,48 @@ public class GfsDataServiceImpl implements IDataService {
return result;
}
+ private ProfileDataset buildProfileDataset(GridDataSet gridDataSet, GfsLevelsEnum levelFlag) {
+ ProfileDataset result = new ProfileDataset();
+ // 利用反射构建数据集
+ Class<? extends ProfileDataset> aClass = result.getClass();
+ for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
+ try {
+ String variableNameInApi = getVariableNameInApi(levelFlag, entry);
+ Grid4D grid4D = entry.getValue();
+ int[] shape = grid4D.getShape();
+ int zSize = shape[1];
+ String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1);
+ Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType());
+ // TODO 2024/6/20: 这里的toArray操作在数据量较大的时候会比较吃内存和cpu
+ float[] values = (float[]) grid4D.toArray().section(
+ new int[]{0, 0, 0, 0}, new int[]{1, zSize, 1, 1}
+ ).copyTo1DJavaArray();
+ method.invoke(result, values);
+ } catch (ReflectiveOperationException e) {
+ throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new AppException(ErrorCode.RESPONSE_DATA_BUILD_ERROR);
+ }
+ }
+ result.setPressureLevels(tableConfig.getPressureList());
+ return result;
+ }
+
+ @NotNull
+ private static String getVariableNameInApi(GfsLevelsEnum levelFlag, Map.Entry<String, Grid4D> variable) {
+ String variableNameInFile = variable.getKey();
+ String variableNameInApi;
+ if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
+ variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
+ } else {
+ variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
+ }
+ if (variableNameInApi == null) {
+ throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
+ }
+ return variableNameInApi;
+ }
private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java
index ce73e73..4e7c2b7 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java
@@ -15,6 +15,8 @@ import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -141,7 +143,9 @@ public class CaiYunServiceImpl implements ISurfaceDataService {
try {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(24);
for (int i = 0; i < 24; i++) {
- timeSeriesDataset.getTime().add(hourly.getTemperature().get(i).getDatetime());
+ timeSeriesDataset.getTime().add(
+ OffsetDateTime.parse(hourly.getTemperature().get(i).getDatetime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)
+ );
timeSeriesDataset.getTemp()[i] = hourly.getTemperature().get(i).getValue();
timeSeriesDataset.getWindSpeed()[i] = (float) MeteoUtils.kmPerHour2mPerSecond(hourly.getWind().get(i).getSpeed());
timeSeriesDataset.getWind360()[i] = hourly.getWind().get(i).getDirection();
diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java
index 4ac416f..7e2eb8e 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java
@@ -20,6 +20,8 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.*;
import static com.htfp.weather.utils.HttpClientUtils.*;
@@ -194,7 +196,9 @@ public class HeFengServiceImpl implements ISurfaceDataService {
List<SurfaceWeatherWarning> surfaceWeatherWarnings = new ArrayList<>();
for (HeFengWarning heFengWarning : heFengWarnings) {
SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning();
- surfaceWeatherWarning.setPubTime(heFengWarning.getPubTime());
+ surfaceWeatherWarning.setPubTime(
+ OffsetDateTime.parse(heFengWarning.getPubTime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)
+ );
// surfaceWeatherWarning.setStartTime(heFengWarning.getStartTime());
surfaceWeatherWarning.setTitle(heFengWarning.getTitle());
surfaceWeatherWarning.setText(heFengWarning.getText());
@@ -213,13 +217,12 @@ public class HeFengServiceImpl implements ISurfaceDataService {
* @return
*/
private BaseHeFengData processHeFengData(BaseHeFengData data) {
- String format = Constant.API_TIME_STRING;
if (data instanceof HeFengForecastHour) {
- String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format);
- ((HeFengForecastHour) data).setFxTime(fxTime);
+ String fxTime = ((HeFengForecastHour) data).getFxTime();
+ ((HeFengForecastHour) data).setFxTime(OffsetDateTime.parse(fxTime).format(Constant.API_TIME_FORMATTER));
} else if (data instanceof HeFengNow) {
- String obsTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengNow) data).getObsTime(), format);
- ((HeFengNow) data).setObsTime(obsTime);
+ String obsTime = ((HeFengNow) data).getObsTime();
+ ((HeFengNow) data).setObsTime(OffsetDateTime.parse(obsTime).format(Constant.API_TIME_FORMATTER));
}
float windSpeed = (float) MeteoUtils.kmPerHour2mPerSecond(data.getWindSpeed());
data.setWindSpeed(windSpeed);
diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java
index 41957c7..83ab309 100644
--- a/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java
+++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java
@@ -1,11 +1,10 @@
package com.htfp.weather.download;
+import com.htfp.weather.download.gfs.GfsDataConfig;
import org.junit.jupiter.api.Test;
import java.io.IOException;
-import static org.junit.jupiter.api.Assertions.*;
-
/**
* @Author : shiyi
* @Date : 2024/4/21 16:28
diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java
index a1e0b2b..8eba0cf 100644
--- a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java
+++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java
@@ -1,11 +1,11 @@
package com.htfp.weather.download;
+import com.htfp.weather.download.gfs.GfsDataConfig;
+import com.htfp.weather.download.gfs.GfsDownloader;
import org.junit.jupiter.api.Test;
import java.util.List;
-import static org.junit.jupiter.api.Assertions.*;
-
/**
* @Author : shiyi
* @Date : 2024/4/25 16:15
diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java
index 71d1b00..1cd50fb 100644
--- a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java
+++ b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java
@@ -1,6 +1,6 @@
package com.htfp.weather.griddata.operation;
-import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
+import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@@ -31,7 +31,6 @@ class GfsDataFetcherTest {
@Test
void getProfile() throws Exception {
log.info("获取廓线");
- gfsDataFetcher.getProfile(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Temp"), 9,0, 0);
}
// @Test
diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java
deleted file mode 100644
index a40da91..0000000
--- a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.htfp.weather.griddata.utils;
-
-import com.htfp.weather.utils.DateTimeUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-import ucar.nc2.NetcdfFile;
-import ucar.nc2.NetcdfFiles;
-
-import java.io.IOException;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-/**
- * @Author : shiyi
- * @Date : 2024/3/4 10:21
- * @Description :
- */
-class GfsUtilsTest {
-
- @Test
- void getRefTime() throws IOException {
- String filename = "D:\\HTFP\\weather\\GFSData\\UTC-20240607.00\\BJT-20240607.17-from-UTC-20240607.00+9.grib2";
- NetcdfFile ncFile = NetcdfFiles.open(filename);
- String refTime = GfsUtils.getRefTime(ncFile);
- String s = filename.split("\\+")[1];
- Integer.parseInt(s.split(".grib2")[0]);
- // assertEquals("2023-09-05T00:00+00:00", refTime);
- // Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN));
- }
-}
\ No newline at end of file