支持地面和近地面数据

refactor
shiyi 10 months ago
parent bc43f9f1ca
commit c950750e67

@ -1,5 +1,6 @@
package com.htfp.weather.download;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.JSONUtils;
@ -73,6 +74,7 @@ public class GfsDataConfig {
@NotEmpty(message = "数据存储路径 (saveRoot) 不能为空")
private String saveRoot;
@JsonIgnore
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json";
public void valid() {
// TODO 2024/4/21: 校验

@ -89,7 +89,7 @@ public class GfsDownloader extends BaseDataDownloader {
log.info("[GFS Download] 文件下载成功: {}", fileOut);
return fileValid(fileOut.getAbsolutePath());
} catch (Exception e) {
log.error("[GFS Download] 文件 {} 下载失败:{}", fileInfo, e.getMessage());
log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
return false;
}
}
@ -116,6 +116,7 @@ public class GfsDownloader extends BaseDataDownloader {
e.printStackTrace();
}
}
log.info("[GFS Download] 下载完成,共 {} 个文件", fileInfoList.size());
return allCompleted;
}
@ -259,6 +260,7 @@ public class GfsDownloader extends BaseDataDownloader {
for (String variable : variables) {
stringJoiner.add(String.format("var_%s=on", variable));
}
stringJoiner.add("lev_surface=on"); // gust需要
return stringJoiner.toString();
}

@ -0,0 +1,55 @@
package com.htfp.weather.griddata.common;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
* @Description :
*/
public enum GfsVariableHeightEnum {
//
TEMP("temp", "Temperature_height_above_ground"),
RH("humidity", "Relative_humidity_cover_height_above_ground"),
WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"),
WIND360("wind360", "Wind_direction_height_above_ground"),
GUST("gustSpeed", "Wind_speed_gust_surface"),
;
private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{
put("temp", "Temperature_height_above_ground");
put("humidity", "Relative_humidity_cover_height_above_ground");
put("windSpeed", "Wind_speed_height_above_ground");
put("wind360", "Wind_direction_height_above_ground");
}};
String nameInApi;
String nameInFile;
GfsVariableHeightEnum(String nameInApi, String nameInFile) {
this.nameInApi = nameInApi;
this.nameInFile = nameInFile;
}
public String getNameInApi() {
return nameInApi;
}
public String getNameInFile() {
return nameInFile;
}
public static String getGfsVariableName(GfsVariableHeightEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);
}
public static List<String> getVariableNamesInApi() {
return new ArrayList<>(variableName.keySet());
}
public static List<String> getVariableNamesInFile() {
return new ArrayList<>(variableName.values());
}
}

@ -9,7 +9,7 @@ import java.util.concurrent.ConcurrentHashMap;
* @Date : 2024/1/15 14:54
* @Description :
*/
public enum GfsVariableNameEnum {
public enum GfsVariableIsobaricEnum {
//
TEMP("temp", "Temperature_isobaric"),
CLOUD("cloud", "Total_cloud_cover_isobaric"),
@ -25,7 +25,7 @@ public enum GfsVariableNameEnum {
String nameInApi;
String nameInFile;
GfsVariableNameEnum(String nameInApi, String nameInFile) {
GfsVariableIsobaricEnum(String nameInApi, String nameInFile) {
this.nameInApi = nameInApi;
this.nameInFile = nameInFile;
}
@ -37,8 +37,8 @@ public enum GfsVariableNameEnum {
public String getNameInFile() {
return nameInFile;
}
public static String getGfsVariableName(GfsVariableNameEnum gfsVariableNameEnum) {
return variableName.get(gfsVariableNameEnum.nameInApi);
public static String getGfsVariableName(GfsVariableIsobaricEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);

@ -2,13 +2,11 @@ package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
@ -19,12 +17,8 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Data @Component @Slf4j
@ -56,7 +50,9 @@ public class TableConfigBean {
@JsonIgnore
public double[] latList;
@JsonIgnore
public int[] levList;
public int[] pressureList;
@JsonIgnore
public int[] heightList;
// @JsonIgnore
// public List<Integer> forecastTimeList; // 预报时效列表
@JsonIgnore
@ -139,6 +135,9 @@ public class TableConfigBean {
}
private void initLevList() {
levList = dataConfig.getPressureLevels();
pressureList = dataConfig.getPressureLevels();
heightList = dataConfig.getHeightLevels();
levSize = pressureList.length + heightList.length;
}
}

@ -43,6 +43,14 @@ public class GfsDataFetcher extends BaseTableOperation {
return gridDataSet;
}
public GridDataSet getSinglePoint(String dataSetId, List<String> variables, int iTime, int iLev, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTime, iLev, iLat, iLon};
int[] shape = new int[] {1, 1, 1, 1};
GridDataSet gridDataSet = queryByTableStore(dataSetId, variables,
origin, shape);
return gridDataSet;
}
/**
*
* @param variable

@ -5,14 +5,13 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.sun.xml.internal.bind.v2.TODO;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;
@ -43,6 +42,7 @@ public class GfsDataImport extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
/**
* init meta data to table store.
*
@ -94,6 +94,7 @@ public class GfsDataImport extends BaseTableOperation {
public void importData(OffsetDateTime refTime) throws Exception {
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES);
@ -103,19 +104,24 @@ public class GfsDataImport extends BaseTableOperation {
int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
// TODO 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
// done 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
// done 2024/5/13: 优化方案使用多线程并使用Future来获取结果但是必须保证每个时刻都成功导入
List<Future<ImportResult>> futures = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
String file = fileList.get(i);
int iTime = i;
futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
// ImportResult importResult = importFromNcFile(meta, file, iTime);
// forecastHours.add(String.valueOf(importResult.getForcastHour()));
}
for (Future<ImportResult> future : futures) {
ImportResult importResult = future.get();
forecastHours.add(String.valueOf(importResult.getForcastHour()));
}
log.info("[tablestore] 数据导入完成, forecastHours: {}", forecastHours);
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
@ -125,12 +131,11 @@ public class GfsDataImport extends BaseTableOperation {
/**
* read data from netcdf file and write data to table store.
*
* @param meta
* @param file netcdf
* @param iTime
* @param meta
* @param file netcdf
* @param iTime
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) {
log.info("[tablestore] 导入文件数据开始,_t={}: {}", iTime, file);
int time = 0;
try {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
@ -138,20 +143,25 @@ public class GfsDataImport extends BaseTableOperation {
// 相对reftime的小时数文件可能缺失因此time可能是不连续的但是iTime是连续的
time = (int) ncFile.findVariable("time").read().getDouble(0);
for (String variableName : meta.getVariables()) {
if ("Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)) {
if ("Wind_speed_gust_surface".equals(variableName) || "Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)
|| "Wind_speed_height_above_ground".equals(variableName) || "Wind_direction_height_above_ground".equals(variableName)) {
continue;
}
Variable variable = ncFile.findVariable(variableName);
if (variable != null) {
for (int z = 0; z < meta.getzSize(); z++) {
// 注意高度索引是递增排序即z=0对应高层低气压值
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variable.getShortName(), iTime, z, grid2D);
}
int shapeLength = variable.getShape().length;
if (shapeLength== 4) {
// 气压坐标和高度坐标保持各自的索引
int zSize = variable.getShape(1);
for (int z = 0; z < zSize; z++) {
// 注意高度索引是递增排序
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variableName, iTime, z, grid2D);
}
} // TODO 2024/5/23: 导入不含高度坐标的数据
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
@ -168,47 +178,58 @@ public class GfsDataImport extends BaseTableOperation {
}
private void transferUnit(String variableName, Array array) {
if (GfsVariableNameEnum.TEMP.getNameInFile().equals(variableName)) {
if (GfsVariableIsobaricEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
if (GfsVariableHeightEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
}
private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向需要保存到文件中
Variable uwnd = ncFile.findVariable("u-component_of_wind_isobaric");
Variable vwnd = ncFile.findVariable("v-component_of_wind_isobaric");
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = meta.getzSize();
if (uwnd == null || vwnd == null) {
return;
}
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
Variable vwnd = ncFile.findVariable("v-component_of_wind" + suffix);
if (uwnd == null || vwnd == null) {
return;
}
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = uwnd.getShape(1);
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray);
Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray);
Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_speed_isobaric", iTime, z, speedGrid2D);
Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray);
Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray);
Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_speed" + suffix, iTime, z, speedGrid2D);
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction_isobaric", iTime, z, dirGrid2D);
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction" + suffix, iTime, z, dirGrid2D);
}
}
}
private List<String> getFileVariables(List<String> variables) {
List<String> fileVariables = new ArrayList<>();
for (String variable : variables) {
String fileVariableName = GfsVariableNameEnum.getGfsVariableName(variable);
if (fileVariableName != null) {
fileVariables.add(fileVariableName);
String variableIsobaric = GfsVariableIsobaricEnum.getGfsVariableName(variable);
if (variableIsobaric != null) {
fileVariables.add(variableIsobaric);
} else {
log.warn("GFS数据文件中气压坐标没有 {} 对应的变量", variable);
}
String variableAboveGround = GfsVariableHeightEnum.getGfsVariableName(variable);
if (variableAboveGround != null) {
fileVariables.add(variableAboveGround);
} else {
log.warn("GFS数据文件中没有 {} 对应的变量", variable);
log.warn("GFS数据文件中(高度坐标)没有 {} 对应的变量", variable);
}
}
return fileVariables;

@ -4,10 +4,7 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.pojo.request.PlaneRequest;
import com.htfp.weather.web.pojo.request.Position3D;
import com.htfp.weather.web.pojo.request.ProfileRequest;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -26,7 +23,7 @@ import java.time.OffsetDateTime;
public class UpperWeatherController {
@Resource(name = "tablestore-gfs")
GfsDataServiceImpl tablestoreService;
GfsDataServiceImpl gfsDataService;
@RequestMapping("/profileByPressure")
public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) {
@ -35,7 +32,7 @@ public class UpperWeatherController {
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
ProfileResponse profileResponse = tablestoreService.getProfile(utcDateTime, variableName, latitude, longitude);
ProfileResponse profileResponse = gfsDataService.getProfile(utcDateTime, variableName, latitude, longitude);
return Result.success(profileResponse);
}
@ -44,7 +41,7 @@ public class UpperWeatherController {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
TimeSeriesDataset forecastSeries = tablestoreService.getForecastSeries(latitude, longitude, level);
TimeSeriesDataset forecastSeries = gfsDataService.getForecastSeries(latitude, longitude, level);
return Result.success(forecastSeries);
}
@ -59,7 +56,16 @@ public class UpperWeatherController {
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
PlaneResponse forecastSeries = tablestoreService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
return Result.success(forecastSeries);
}
@PostMapping("/now")
public Result queryNowWeather(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
NowWeatherStatus nowWeather = gfsDataService.getNowWeather(latitude, longitude, level);
return Result.success(nowWeather);
}
}

@ -8,8 +8,8 @@ package com.htfp.weather.web.exception;
public enum ErrorCode {
//
VALIDATE_ERROR(1001, "参数校验错误"),
CONFIG_ERROR(1002, "配置相关错误"),
VALIDATE_ERROR(1001, "参数校验错误 "),
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),

@ -1,8 +1,7 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
@ -13,7 +12,6 @@ import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
/**
* @Author : shiyi
@ -27,7 +25,7 @@ public class PlaneRequest {
String time;
@NotNull(message = "变量名不能为空")
@EnumValid(enumClass = GfsVariableNameEnum.class, usedField = "getNameInApi", message = "变量不存在")
@EnumValid(enumClass = GfsVariableIsobaricEnum.class, usedField = "getNameInApi", message = "变量不存在")
String variableName;
@NotNull(message = "高度层不能为空")

@ -4,7 +4,8 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.QueryMeta;
@ -13,6 +14,7 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
@ -37,6 +39,7 @@ import java.util.stream.Collectors;
*/
@Service("tablestore-gfs") @Slf4j
public class GfsDataServiceImpl implements IDataService {
private static final int SURFACE_LEVEL = 9999;
@Resource
GfsDataFetcher gfsDataFetcher;
@Resource
@ -46,7 +49,7 @@ public class GfsDataServiceImpl implements IDataService {
public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) {
String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName);
String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
if (targetVariable == null) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
@ -58,12 +61,12 @@ public class GfsDataServiceImpl implements IDataService {
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTimeIndex(targetTime, lastGridDataSetMeta);
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
int[] pressureLevels = tableConfigBean.getLevList();
int[] pressureLevels = tableConfigBean.getPressureList();
Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon);
float[] values = (float[]) array.getStorage();
return new ProfileResponse(pressureLevels, values);
@ -73,52 +76,94 @@ public class GfsDataServiceImpl implements IDataService {
}
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int pressureLevel) {
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
OffsetDateTime currentTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now());
if (currentTime.isBefore(refTime)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
int startHour = getForecastHourFromTargetTime(currentTime, refTime);
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
List<Integer> iTimeList = new ArrayList<>();
List<String> timeList = new ArrayList<>();
Map<String, List> params = getFutureTimeIndexList(lastGridDataSetMeta);
List<Integer> iTimeList = params.get("iTimeList");
List<String> timeList = params.get("timeList");
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray();
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level);
for (int i = 0; i < forecastHours.size(); i++) {
int hour = Integer.parseInt(forecastHours.get(i));
if (hour >= startHour) {
iTimeList.add(i);
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
timeList.add(timeStr);
}
if (iTimeList.size() >= 24) {
break;
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildTimeSeriesDatasetSurface(timeList, gridDataSet);
} else {
return buildTimeSeriesDatasetPressure(timeList, gridDataSet);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray();
}
public NowWeatherStatus getNowWeather(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(pressureLevel);
List<String> variableNames = GfsVariableNameEnum.getVariableNamesInFile();
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
return buildTimeSeriesDataset(timeList, gridDataSet);
GridDataSet gridDataSet = gfsDataFetcher.getSinglePoint(gridDataSetId, variableNames, iTime, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildNowWeatherSurface(gridDataSet);
} else {
return buildNowWeatherPressure(gridDataSet);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private List<String> getVariableNamesInDatabase(int level) {
List<String> variableNames = new ArrayList<>();
if (level == SURFACE_LEVEL) {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
}
}
} else {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
}
}
}
return variableNames;
}
public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) {
String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName);
int iLev;
String targetVariable;
if (level == SURFACE_LEVEL) {
iLev = 0; // NOTE 2024/5/20: 0 表示地面风速对应10m高度其他变量对应2m高度
targetVariable = GfsVariableHeightEnum.getGfsVariableName(variableName);
} else {
iLev = getPressureIndex(level);
targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
}
if (targetVariable == null) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
@ -130,8 +175,7 @@ public class GfsDataServiceImpl implements IDataService {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTimeIndex(targetTime, lastGridDataSetMeta);
int iLev = getLevelIndex(level);
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta);
int iMinLat = getLatitudeIndex(minLat);
int iMaxLat = getLatitudeIndex(maxLat);
int iMinLon = getLongitudeIndex(minLon);
@ -159,30 +203,92 @@ public class GfsDataServiceImpl implements IDataService {
return planeResponse;
}
private TimeSeriesDataset buildTimeSeriesDataset(List<String> timeList, GridDataSet gridDataSet) {
private TimeSeriesDataset buildTimeSeriesDatasetPressure(List<String> timeList, GridDataSet gridDataSet) {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(timeList.size());
timeSeriesDataset.setTime(timeList);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.TEMP));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP));
if (grid4D != null) {
timeSeriesDataset.setTemp((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND_SPEED));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED));
if (grid4D != null) {
timeSeriesDataset.setWindSpeed((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND360));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360));
if (grid4D != null) {
timeSeriesDataset.setWind360((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.CLOUD));
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD));
if (grid4D != null) {
timeSeriesDataset.setCloud((float[]) grid4D.toArray().getStorage());
}
return timeSeriesDataset;
}
private int getTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
private TimeSeriesDataset buildTimeSeriesDatasetSurface(List<String> timeList, GridDataSet gridDataSet) {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(timeList.size());
timeSeriesDataset.setTime(timeList);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP));
if (grid4D != null) {
timeSeriesDataset.setTemp((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED));
if (grid4D != null) {
timeSeriesDataset.setWindSpeed((float[]) grid4D.toArray().getStorage());
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360));
if (grid4D != null) {
timeSeriesDataset.setWind360((float[]) grid4D.toArray().getStorage());
}
return timeSeriesDataset;
}
private NowWeatherStatus buildNowWeatherPressure(GridDataSet gridDataSet) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD));
if (grid4D != null) {
nowWeatherStatus.setCloud(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
}
private NowWeatherStatus buildNowWeatherSurface(GridDataSet gridDataSet) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
}
private int getTargetTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
if (targetTime.isBefore(refTime)) {
@ -198,6 +304,38 @@ public class GfsDataServiceImpl implements IDataService {
return iTime;
}
private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
OffsetDateTime currentTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now());
if (currentTime.isBefore(refTime)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
int startHour = getForecastHourFromTargetTime(currentTime, refTime);
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
List<Integer> iTimeList = new ArrayList<>();
List<String> timeList = new ArrayList<>();
for (int i = 0; i < forecastHours.size(); i++) {
int hour = Integer.parseInt(forecastHours.get(i));
if (hour >= startHour) {
iTimeList.add(i);
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
timeList.add(timeStr);
}
if (iTimeList.size() >= 24) {
break;
}
}
if (CollectionUtils.isEmpty(iTimeList) || CollectionUtils.isEmpty(timeList)) {
throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR);
}
Map<String, List> map = new HashMap<>(2);
map.put("iTimeList", iTimeList);
map.put("timeList", timeList);
return map;
}
private int getLongitudeIndex(double lon) {
double[] lonList = tableConfigBean.getLonList();
if (lonList.length == 0) {
@ -233,12 +371,24 @@ public class GfsDataServiceImpl implements IDataService {
return nearestIndex;
}
private int getLevelIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfigBean.getLevList()).boxed().collect(Collectors.toList());
private int getPressureIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfigBean.getPressureList()).boxed().collect(Collectors.toList());
if (CollectionUtils.isEmpty(levList)) {
throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "经度列表为空, 无法查找索引");
throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空, 无法查找索引");
}
int index = levList.indexOf(pressureLevel);
if (index == -1) {
throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + pressureLevel + "的气压高度索引");
}
return index;
}
private int getLevelIndex(int level) {
if (level == SURFACE_LEVEL) {
return 0; // NOTE 2024/5/20: 0 表示地面风速对应10m高度其他变量对应2m高度
} else {
return getPressureIndex(level);
}
return levList.indexOf(pressureLevel);
}
private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) {

@ -1,11 +1,11 @@
{
"duration" : 10,
"duration" : 30,
"minLon" : 70.0,
"maxLon" : 140.0,
"minLat" : 0.0,
"maxLat" : 55.0,
"resolution" : 0.25,
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ],
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST"],
"pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000],
"saveRoot" : "GFSData"
}

@ -1,6 +1,6 @@
package com.htfp.weather.griddata.operation;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@ -25,13 +25,13 @@ class GfsDataFetcherTest {
@Test
void getPlane() throws Exception {
log.info("获取平面");
gfsDataFetcher.getPlane(dataSetId, GfsVariableNameEnum.getGfsVariableName("Cloud"),0, 0, 10, 10, 10, 10);
gfsDataFetcher.getPlane(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Cloud"),0, 0, 10, 10, 10, 10);
}
@Test
void getProfile() throws Exception {
log.info("获取廓线");
gfsDataFetcher.getProfile(dataSetId, GfsVariableNameEnum.getGfsVariableName("Temp"), 9,0, 0);
gfsDataFetcher.getProfile(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Temp"), 9,0, 0);
}
// @Test

Loading…
Cancel
Save