504-Software-Development-Group
/
WeatherService
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

支持地面和近地面数据

Browse Source
refactor
shiyi 10 months ago
parent bc43f9f1ca
commit c950750e67
13 changed files with 361 additions and 120 deletions
Show Stats Download Patch File Download Diff File

2
weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java
Unescape Escape View File

@ -1,5 +1,6 @@
package com.htfp.weather.download;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.JSONUtils;
@ -73,6 +74,7 @@ public class GfsDataConfig {
@NotEmpty(message = "数据存储路径 (saveRoot) 不能为空")
private String saveRoot;
@JsonIgnore
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json";
public void valid() {
// TODO 2024/4/21: 校验

4
weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
Unescape Escape View File

@ -89,7 +89,7 @@ public class GfsDownloader extends BaseDataDownloader {
log.info("[GFS Download] 文件下载成功: {}", fileOut);
return fileValid(fileOut.getAbsolutePath());
} catch (Exception e) {
log.error("[GFS Download] 文件 {} 下载失败：{}", fileInfo, e.getMessage());
log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
return false;
}
}
@ -116,6 +116,7 @@ public class GfsDownloader extends BaseDataDownloader {
e.printStackTrace();
}
}
log.info("[GFS Download] 下载完成，共 {} 个文件", fileInfoList.size());
return allCompleted;
}
@ -259,6 +260,7 @@ public class GfsDownloader extends BaseDataDownloader {
for (String variable : variables) {
stringJoiner.add(String.format("var_%s=on", variable));
}
stringJoiner.add("lev_surface=on"); // gust需要
return stringJoiner.toString();
}

55
weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableHeightEnum.java
Unescape Escape View File

@ -0,0 +1,55 @@
package com.htfp.weather.griddata.common;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
* @Description : 文件变量名和表中变量名的映射
*/
public enum GfsVariableHeightEnum {
//
TEMP("temp", "Temperature_height_above_ground"),
RH("humidity", "Relative_humidity_cover_height_above_ground"),
WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"),
WIND360("wind360", "Wind_direction_height_above_ground"),
GUST("gustSpeed", "Wind_speed_gust_surface"),
;
private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{
put("temp", "Temperature_height_above_ground");
put("humidity", "Relative_humidity_cover_height_above_ground");
put("windSpeed", "Wind_speed_height_above_ground");
put("wind360", "Wind_direction_height_above_ground");
}};
String nameInApi;
String nameInFile;
GfsVariableHeightEnum(String nameInApi, String nameInFile) {
this.nameInApi = nameInApi;
this.nameInFile = nameInFile;
}
public String getNameInApi() {
return nameInApi;
}
public String getNameInFile() {
return nameInFile;
}
public static String getGfsVariableName(GfsVariableHeightEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);
}
public static List<String> getVariableNamesInApi() {
return new ArrayList<>(variableName.keySet());
}
public static List<String> getVariableNamesInFile() {
return new ArrayList<>(variableName.values());
}
}

8
weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java → weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableIsobaricEnum.java
Unescape Escape View File

@ -9,7 +9,7 @@ import java.util.concurrent.ConcurrentHashMap;
* @Date : 2024/1/15 14:54
* @Description : 文件变量名和表中变量名的映射
*/
public enum GfsVariableNameEnum {
public enum GfsVariableIsobaricEnum {
//
TEMP("temp", "Temperature_isobaric"),
CLOUD("cloud", "Total_cloud_cover_isobaric"),
@ -25,7 +25,7 @@ public enum GfsVariableNameEnum {
String nameInApi;
String nameInFile;
GfsVariableNameEnum(String nameInApi, String nameInFile) {
GfsVariableIsobaricEnum(String nameInApi, String nameInFile) {
this.nameInApi = nameInApi;
this.nameInFile = nameInFile;
}
@ -37,8 +37,8 @@ public enum GfsVariableNameEnum {
public String getNameInFile() {
return nameInFile;
}
public static String getGfsVariableName(GfsVariableNameEnum gfsVariableNameEnum) {
return variableName.get(gfsVariableNameEnum.nameInApi);
public static String getGfsVariableName(GfsVariableIsobaricEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);

15
weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java
Unescape Escape View File

@ -2,13 +2,11 @@ package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
@ -19,12 +17,8 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Data @Component @Slf4j
@ -56,7 +50,9 @@ public class TableConfigBean {
@JsonIgnore
public double[] latList;
@JsonIgnore
public int[] levList;
public int[] pressureList;
@JsonIgnore
public int[] heightList;
// @JsonIgnore
// public List<Integer> forecastTimeList; // 预报时效列表
@JsonIgnore
@ -139,6 +135,9 @@ public class TableConfigBean {
}
private void initLevList() {
levList = dataConfig.getPressureLevels();
pressureList = dataConfig.getPressureLevels();
heightList = dataConfig.getHeightLevels();
levSize = pressureList.length + heightList.length;
}
}

8
weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java
Unescape Escape View File

@ -43,6 +43,14 @@ public class GfsDataFetcher extends BaseTableOperation {
return gridDataSet;
}
public GridDataSet getSinglePoint(String dataSetId, List<String> variables, int iTime, int iLev, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTime, iLev, iLat, iLon};
int[] shape = new int[] {1, 1, 1, 1};
GridDataSet gridDataSet = queryByTableStore(dataSetId, variables,
origin, shape);
return gridDataSet;
}
/**
* 获取指定变量的二维全平面数据
* @param variable 变量名

111
weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java
Unescape Escape View File

@ -5,14 +5,13 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.sun.xml.internal.bind.v2.TODO;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;
@ -43,6 +42,7 @@ public class GfsDataImport extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
/**
* init meta data to table store.
*
@ -94,6 +94,7 @@ public class GfsDataImport extends BaseTableOperation {
public void importData(OffsetDateTime refTime) throws Exception {
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES);
@ -103,19 +104,24 @@ public class GfsDataImport extends BaseTableOperation {
int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
// TODO 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长
// done 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长
// done 2024/5/13: 优化方案，使用多线程，并使用Future来获取结果，但是必须保证每个时刻都成功导入
List<Future<ImportResult>> futures = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
String file = fileList.get(i);
int iTime = i;
futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
// ImportResult importResult = importFromNcFile(meta, file, iTime);
// forecastHours.add(String.valueOf(importResult.getForcastHour()));
}
for (Future<ImportResult> future : futures) {
ImportResult importResult = future.get();
forecastHours.add(String.valueOf(importResult.getForcastHour()));
}
log.info("[tablestore] 数据导入完成, forecastHours: {}", forecastHours);
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
@ -125,12 +131,11 @@ public class GfsDataImport extends BaseTableOperation {
/**
* read data from netcdf file and write data to table store.
*
* @param meta 数据元信息
* @param file netcdf文件路径
* @param iTime 文件序号
* @param meta 数据元信息
* @param file netcdf文件路径
* @param iTime 文件序号
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) {
log.info("[tablestore] 导入文件数据开始,_t={}: {}", iTime, file);
int time = 0;
try {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
@ -138,20 +143,25 @@ public class GfsDataImport extends BaseTableOperation {
// 相对reftime的小时数，文件可能缺失，因此time可能是不连续的，但是iTime是连续的
time = (int) ncFile.findVariable("time").read().getDouble(0);
for (String variableName : meta.getVariables()) {
if ("Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)) {
if ("Wind_speed_gust_surface".equals(variableName) || "Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)
|| "Wind_speed_height_above_ground".equals(variableName) || "Wind_direction_height_above_ground".equals(variableName)) {
continue;
}
Variable variable = ncFile.findVariable(variableName);
if (variable != null) {
for (int z = 0; z < meta.getzSize(); z++) {
// 注意高度索引是递增排序，即z=0对应高层低气压值
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variable.getShortName(), iTime, z, grid2D);
}
int shapeLength = variable.getShape().length;
if (shapeLength== 4) {
// 气压坐标和高度坐标保持各自的索引
int zSize = variable.getShape(1);
for (int z = 0; z < zSize; z++) {
// 注意高度索引是递增排序
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variableName, iTime, z, grid2D);
}
} // TODO 2024/5/23: 导入不含高度坐标的数据
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
@ -168,47 +178,58 @@ public class GfsDataImport extends BaseTableOperation {
}
private void transferUnit(String variableName, Array array) {
if (GfsVariableNameEnum.TEMP.getNameInFile().equals(variableName)) {
if (GfsVariableIsobaricEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
if (GfsVariableHeightEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
}
private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向需要保存到文件中
Variable uwnd = ncFile.findVariable("u-component_of_wind_isobaric");
Variable vwnd = ncFile.findVariable("v-component_of_wind_isobaric");
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = meta.getzSize();
if (uwnd == null || vwnd == null) {
return;
}
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
Variable vwnd = ncFile.findVariable("v-component_of_wind" + suffix);
if (uwnd == null || vwnd == null) {
return;
}
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = uwnd.getShape(1);
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray);
Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray);
Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_speed_isobaric", iTime, z, speedGrid2D);
Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray);
Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray);
Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_speed" + suffix, iTime, z, speedGrid2D);
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction_isobaric", iTime, z, dirGrid2D);
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction" + suffix, iTime, z, dirGrid2D);
}
}
}
private List<String> getFileVariables(List<String> variables) {
List<String> fileVariables = new ArrayList<>();
for (String variable : variables) {
String fileVariableName = GfsVariableNameEnum.getGfsVariableName(variable);
if (fileVariableName != null) {
fileVariables.add(fileVariableName);
String variableIsobaric = GfsVariableIsobaricEnum.getGfsVariableName(variable);
if (variableIsobaric != null) {
fileVariables.add(variableIsobaric);
} else {
log.warn("GFS数据文件中（气压坐标）没有 {} 对应的变量", variable);
}
String variableAboveGround = GfsVariableHeightEnum.getGfsVariableName(variable);
if (variableAboveGround != null) {
fileVariables.add(variableAboveGround);
} else {
log.warn("GFS数据文件中没有 {} 对应的变量", variable);
log.warn("GFS数据文件中（高度坐标）没有 {} 对应的变量", variable);
}
}
return fileVariables;

22
weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java
Unescape Escape View File

@ -4,10 +4,7 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.pojo.request.PlaneRequest;
import com.htfp.weather.web.pojo.request.Position3D;
import com.htfp.weather.web.pojo.request.ProfileRequest;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -26,7 +23,7 @@ import java.time.OffsetDateTime;
public class UpperWeatherController {
@Resource(name = "tablestore-gfs")
GfsDataServiceImpl tablestoreService;
GfsDataServiceImpl gfsDataService;
@RequestMapping("/profileByPressure")
public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) {
@ -35,7 +32,7 @@ public class UpperWeatherController {
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
ProfileResponse profileResponse = tablestoreService.getProfile(utcDateTime, variableName, latitude, longitude);
ProfileResponse profileResponse = gfsDataService.getProfile(utcDateTime, variableName, latitude, longitude);
return Result.success(profileResponse);
}
@ -44,7 +41,7 @@ public class UpperWeatherController {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
TimeSeriesDataset forecastSeries = tablestoreService.getForecastSeries(latitude, longitude, level);
TimeSeriesDataset forecastSeries = gfsDataService.getForecastSeries(latitude, longitude, level);
return Result.success(forecastSeries);
}
@ -59,7 +56,16 @@ public class UpperWeatherController {
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
PlaneResponse forecastSeries = tablestoreService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
return Result.success(forecastSeries);
}
@PostMapping("/now")
public Result queryNowWeather(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
NowWeatherStatus nowWeather = gfsDataService.getNowWeather(latitude, longitude, level);
return Result.success(nowWeather);
}
}

4
weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
Unescape Escape View File

@ -8,8 +8,8 @@ package com.htfp.weather.web.exception;
public enum ErrorCode {
//
VALIDATE_ERROR(1001, "参数校验错误"),
CONFIG_ERROR(1002, "配置相关错误"),
VALIDATE_ERROR(1001, "参数校验错误 "),
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),

6
weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java
Unescape Escape View File

@ -1,8 +1,7 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
@ -13,7 +12,6 @@ import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
/**
* @Author : shiyi
@ -27,7 +25,7 @@ public class PlaneRequest {
String time;
@NotNull(message = "变量名不能为空")
@EnumValid(enumClass = GfsVariableNameEnum.class, usedField = "getNameInApi", message = "变量不存在")
@EnumValid(enumClass = GfsVariableIsobaricEnum.class, usedField = "getNameInApi", message = "变量不存在")
String variableName;
@NotNull(message = "高度层不能为空")

236
weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java
Unescape Escape View File

@ -4,7 +4,8 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.QueryMeta;
@ -13,6 +14,7 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
@ -37,6 +39,7 @@ import java.util.stream.Collectors;
*/
@Service("tablestore-gfs") @Slf4j
public class GfsDataServiceImpl implements IDataService {
private static final int SURFACE_LEVEL = 9999;
@Resource
GfsDataFetcher gfsDataFetcher;
@Resource
@ -46,7 +49,7 @@ public class GfsDataServiceImpl implements IDataService {
public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) {
String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName);
String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
if (targetVariable == null) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
@ -58,12 +61,12 @@ public class GfsDataServiceImpl implements IDataService {
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTimeIndex(targetTime, lastGridDataSetMeta);
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
int[] pressureLevels = tableConfigBean.getLevList();
int[] pressureLevels = tableConfigBean.getPressureList();
Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon);
float[] values = (float[]) array.getStorage();
return new ProfileResponse(pressureLevels, values);
@ -73,52 +76,94 @@ public class GfsDataServiceImpl implements IDataService {
}
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int pressureLevel) {
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
OffsetDateTime currentTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now());
if (currentTime.isBefore(refTime)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
int startHour = getForecastHourFromTargetTime(currentTime, refTime);
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
List<Integer> iTimeList = new ArrayList<>();
List<String> timeList = new ArrayList<>();
Map<String, List> params = getFutureTimeIndexList(lastGridDataSetMeta);
List<Integer> iTimeList = params.get("iTimeList");
List<String> timeList = params.get("timeList");
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray();
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level);
for (int i = 0; i < forecastHours.size(); i++) {
int hour = Integer.parseInt(forecastHours.get(i));
if (hour >= startHour) {
iTimeList.add(i);
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
timeList.add(timeStr);
}
if (iTimeList.size() >= 24) {
break;
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildTimeSeriesDatasetSurface(timeList, gridDataSet);
} else {
return buildTimeSeriesDatasetPressure(timeList, gridDataSet);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray();
}
public NowWeatherStatus getNowWeather(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(pressureLevel);
List<String> variableNames = GfsVariableNameEnum.getVariableNamesInFile();
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
return buildTimeSeriesDataset(timeList, gridDataSet);
GridDataSet gridDataSet = gfsDataFetcher.getSinglePoint(gridDataSetId, variableNames, iTime, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildNowWeatherSurface(gridDataSet);
} else {
return buildNowWeatherPressure(gridDataSet);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private List<String> getVariableNamesInDatabase(int level) {
List<String> variableNames = new ArrayList<>();
if (level == SURFACE_LEVEL) {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
}
}
} else {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
}
}
}
return variableNames;
}
public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) {
String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName);
int iLev;
String targetVariable;
if (level == SURFACE_LEVEL) {
iLev = 0; // NOTE 2024/5/20: 0 表示地面，风速对应10m高度，其他变量对应2m高度
targetVariable = GfsVariableHeightEnum.getGfsVariableName(variableName);
} else {
iLev = getPressureIndex(level);
targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
}
if (targetVariable == null) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
@ -130,8 +175,7 @@ public class GfsDataServiceImpl implements IDataService {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTimeIndex(targetTime, lastGridDataSetMeta);
int iLev = getLevelIndex(level);
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta);
int iMinLat = getLatitudeIndex(minLat);
int iMaxLat = getLatitudeIndex(maxLat);
int iMinLon = getLongitudeIndex(minLon);
@ -159,30 +203,92 @@ public class GfsDataServiceImpl implements IDataService {
return planeResponse;
}
private TimeSeriesDataset buildTimeSeriesDataset(List<String> timeList, GridDataSet gridDataSet) {
private TimeSeriesDataset buildTimeSeriesDatasetPressure(List<String> timeList, GridDataSet gridDataSet) {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(timeList.size());
timeSeriesDataset.setTime(timeList);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.TEMP));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP));
if (grid4D != null) {
timeSeriesDataset.setTemp((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND_SPEED));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED));
if (grid4D != null) {
timeSeriesDataset.setWindSpeed((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND360));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360));
if (grid4D != null) {
timeSeriesDataset.setWind360((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.CLOUD));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD));
if (grid4D != null) {
timeSeriesDataset.setCloud((float[]) grid4D.toArray().getStorage());
}
return timeSeriesDataset;
}
private int getTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
private TimeSeriesDataset buildTimeSeriesDatasetSurface(List<String> timeList, GridDataSet gridDataSet) {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(timeList.size());
timeSeriesDataset.setTime(timeList);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP));
if (grid4D != null) {
timeSeriesDataset.setTemp((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED));
if (grid4D != null) {
timeSeriesDataset.setWindSpeed((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360));
if (grid4D != null) {
timeSeriesDataset.setWind360((float[]) grid4D.toArray().getStorage());
}
return timeSeriesDataset;
}
private NowWeatherStatus buildNowWeatherPressure(GridDataSet gridDataSet) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD));
if (grid4D != null) {
nowWeatherStatus.setCloud(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
}
private NowWeatherStatus buildNowWeatherSurface(GridDataSet gridDataSet) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
}
private int getTargetTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
if (targetTime.isBefore(refTime)) {
@ -198,6 +304,38 @@ public class GfsDataServiceImpl implements IDataService {
return iTime;
}
private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
OffsetDateTime currentTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now());
if (currentTime.isBefore(refTime)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
int startHour = getForecastHourFromTargetTime(currentTime, refTime);
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
List<Integer> iTimeList = new ArrayList<>();
List<String> timeList = new ArrayList<>();
for (int i = 0; i < forecastHours.size(); i++) {
int hour = Integer.parseInt(forecastHours.get(i));
if (hour >= startHour) {
iTimeList.add(i);
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
timeList.add(timeStr);
}
if (iTimeList.size() >= 24) {
break;
}
}
if (CollectionUtils.isEmpty(iTimeList) || CollectionUtils.isEmpty(timeList)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
Map<String, List> map = new HashMap<>(2);
map.put("iTimeList", iTimeList);
map.put("timeList", timeList);
return map;
}
private int getLongitudeIndex(double lon) {
double[] lonList = tableConfigBean.getLonList();
if (lonList.length == 0) {
@ -233,12 +371,24 @@ public class GfsDataServiceImpl implements IDataService {
return nearestIndex;
}
private int getLevelIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfigBean.getLevList()).boxed().collect(Collectors.toList());
private int getPressureIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfigBean.getPressureList()).boxed().collect(Collectors.toList());
if (CollectionUtils.isEmpty(levList)) {
throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "经度列表为空， 无法查找索引");
throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空， 无法查找索引");
}
int index = levList.indexOf(pressureLevel);
if (index == -1) {
throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + pressureLevel + "的气压高度索引");
}
return index;
}
private int getLevelIndex(int level) {
if (level == SURFACE_LEVEL) {
return 0; // NOTE 2024/5/20: 0 表示地面，风速对应10m高度，其他变量对应2m高度
} else {
return getPressureIndex(level);
}
return levList.indexOf(pressureLevel);
}
private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) {

4
weather-service/src/main/resources/config/gfsDataConfig.json
Unescape Escape View File

@ -1,11 +1,11 @@
{
"duration" : 10,
"duration" : 30,
"minLon" : 70.0,
"maxLon" : 140.0,
"minLat" : 0.0,
"maxLat" : 55.0,
"resolution" : 0.25,
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ],
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST"],
"pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000],
"saveRoot" : "GFSData"
}

6
weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java
Unescape Escape View File

@ -1,6 +1,6 @@
package com.htfp.weather.griddata.operation;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@ -25,13 +25,13 @@ class GfsDataFetcherTest {
@Test
void getPlane() throws Exception {
log.info("获取平面");
gfsDataFetcher.getPlane(dataSetId, GfsVariableNameEnum.getGfsVariableName("Cloud"),0, 0, 10, 10, 10, 10);
gfsDataFetcher.getPlane(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Cloud"),0, 0, 10, 10, 10, 10);
}
@Test
void getProfile() throws Exception {
log.info("获取廓线");
gfsDataFetcher.getProfile(dataSetId, GfsVariableNameEnum.getGfsVariableName("Temp"), 9,0, 0);
gfsDataFetcher.getProfile(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Temp"), 9,0, 0);
}
// @Test

Write Preview
Loading…
Cancel
Save