查询优化:最新的meta数据定期查询维护,查询数据时不查询meta;多航点查询时,只查航线所在区域;序号索引放在fetch操作层,service层只考虑经纬度

refactor
shiyi 9 months ago
parent 4864bd2fbd
commit 8357fac67f

@ -10,7 +10,7 @@ public class TableStoreGridConfig {
private String metaTableName; private String metaTableName;
private String dataTableName; private String dataTableName;
private long dataSizeLimitForFetch = 20 * 1024 * 1024; private long dataSizeLimitForFetch = 24 * 1024 * 1024;
public String getTableStoreEndpoint() { public String getTableStoreEndpoint() {
return tableStoreEndpoint; return tableStoreEndpoint;

@ -1,11 +1,13 @@
package com.aliyun.tablestore.grid.model; package com.aliyun.tablestore.grid.model;
import lombok.ToString;
import ucar.ma2.DataType; import ucar.ma2.DataType;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ToString
public class GridDataSetMeta { public class GridDataSetMeta {
private String gridDataSetId; private String gridDataSetId;

@ -1,6 +1,8 @@
package com.aliyun.tablestore.grid.model.grid; package com.aliyun.tablestore.grid.model.grid;
import org.checkerframework.checker.units.qual.A;
import ucar.ma2.DataType; import ucar.ma2.DataType;
import ucar.ma2.Index;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Arrays; import java.util.Arrays;
@ -27,4 +29,36 @@ public class Grid4D extends Grid {
return new Grid3D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length), return new Grid3D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length),
Arrays.copyOfRange(shape, 1, shape.length)); Arrays.copyOfRange(shape, 1, shape.length));
} }
public float[][][][] section(int[] origin, int[] shape) {
// // calculate rank
// int rank = 0;
// for (int size : shape) {
// if (size > 1) {
// rank++;
// }
// }
// buffer.getShort()
// switch (rank) {
// case 4: {
// float[][][][] result = new float[shape[0]][shape[1]][shape[2]][shape[3]];
// Index index = Index.factory(shape);
// index.currentElement();
// for (int i = 0; i < shape[0]; i++)
// float[][][] grid3D = getGrid3D(i).section(Arrays.copyOfRange(origin, 1, origin.length),
// Arrays.copyOfRange(shape, 1, shape.length));
// for (int j = 0; j < shape[1]; j++)
// break;
// }
// default:
// break;
// }
// // calculate stride
// int[] stride = new int[4];
// stride[3] = 1;
// for (int i = 2; i >= 0; i--) {
// stride[i] = stride[i + 1] * shape[i + 1];
// }
return null;
}
} }

@ -5,7 +5,7 @@
"minLat" : 0.0, "minLat" : 0.0,
"maxLat" : 55.0, "maxLat" : 55.0,
"resolution" : 0.25, "resolution" : 0.25,
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ], "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC", "PRATE" ],
"pressureLevels" : [ 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000 ], "pressureLevels" : [ 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000 ],
"heightLevels" : [ 2, 10, 20, 30, 40, 50, 80, 100 ], "heightLevels" : [ 2, 10, 20, 30, 40, 50, 80, 100 ],
"saveRoot" : "./GFSData" "saveRoot" : "./GFSData"

@ -261,6 +261,7 @@ public class GfsDownloader extends BaseDataDownloader {
for (int height : heightLevels) { for (int height : heightLevels) {
stringJoiner.add(String.format("lev_%d_m_above_ground", height)); stringJoiner.add(String.format("lev_%d_m_above_ground", height));
} }
stringJoiner.add("lev_surface=on");
return stringJoiner.toString(); return stringJoiner.toString();
} }

@ -12,7 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
public enum GfsVariableHeightEnum { public enum GfsVariableHeightEnum {
// //
TEMP("temp", "Temperature_height_above_ground"), TEMP("temp", "Temperature_height_above_ground"),
RH("humidity", "Relative_humidity_cover_height_above_ground"), RH("humidity", "Relative_humidity_height_above_ground"),
WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"), WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"),
WIND360("wind360", "Wind_direction_height_above_ground"), WIND360("wind360", "Wind_direction_height_above_ground"),
// GUST("gustSpeed", "Wind_speed_gust_surface"), // GUST("gustSpeed", "Wind_speed_gust_surface"),
@ -20,7 +20,7 @@ public enum GfsVariableHeightEnum {
; ;
private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{ private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{
put("temp", "Temperature_height_above_ground"); put("temp", "Temperature_height_above_ground");
put("humidity", "Relative_humidity_cover_height_above_ground"); put("humidity", "Relative_humidity_height_above_ground");
put("windSpeed", "Wind_speed_height_above_ground"); put("windSpeed", "Wind_speed_height_above_ground");
put("wind360", "Wind_direction_height_above_ground"); put("wind360", "Wind_direction_height_above_ground");
put("precip", "Precipitation_rate_surface"); put("precip", "Precipitation_rate_surface");

@ -12,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
public enum GfsVariableIsobaricEnum { public enum GfsVariableIsobaricEnum {
// //
TEMP("temp", "Temperature_isobaric"), TEMP("temp", "Temperature_isobaric"),
RH("humidity", "Relative_humidity_isobaric"),
CLOUD("cloud", "Total_cloud_cover_isobaric"), CLOUD("cloud", "Total_cloud_cover_isobaric"),
WIND_SPEED("windSpeed", "Wind_speed_isobaric"), WIND_SPEED("windSpeed", "Wind_speed_isobaric"),
WIND360("wind360", "Wind_direction_isobaric"), WIND360("wind360", "Wind_direction_isobaric"),
@ -21,6 +22,7 @@ public enum GfsVariableIsobaricEnum {
put("cloud", "Total_cloud_cover_isobaric"); put("cloud", "Total_cloud_cover_isobaric");
put("windSpeed", "Wind_speed_isobaric"); put("windSpeed", "Wind_speed_isobaric");
put("wind360", "Wind_direction_isobaric"); put("wind360", "Wind_direction_isobaric");
put("humidity", "Relative_humidity_isobaric");
}}; }};
String nameInApi; String nameInApi;
String nameInFile; String nameInFile;

@ -137,7 +137,6 @@ public class TableConfig {
private void initLevList() { private void initLevList() {
pressureList = dataConfig.getPressureLevels(); pressureList = dataConfig.getPressureLevels();
heightList = dataConfig.getHeightLevels(); heightList = dataConfig.getHeightLevels();
// this.levSize = Math.max(pressureList.length, heightList.length);
levSize = pressureList.length + heightList.length;
} }
} }

@ -1,17 +1,34 @@
package com.htfp.weather.griddata.operation; package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.GridDataFetcher; import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet; import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Range;
import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.utils.ValueRange;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.DependsOn;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import ucar.ma2.Array; import ucar.ma2.Array;
import javax.annotation.PostConstruct;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.time.Duration;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* @Author : shiyi * @Author : shiyi
@ -23,6 +40,31 @@ import java.util.List;
public class GfsDataFetcher extends BaseTableOperation { public class GfsDataFetcher extends BaseTableOperation {
@Resource @Resource
TableConfig tableConfig; TableConfig tableConfig;
@Resource
QueryMeta queryMeta;
GridDataSetMeta lastGridDataSetMeta;
public GridDataSetMeta getLastGridDataSetMeta() {
if (lastGridDataSetMeta == null) {
updateLastGridDataSetMeta();
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
return lastGridDataSetMeta;
}
/**
*
*/
@Scheduled(cron = "0 30 * * * ?") @PostConstruct
public GridDataSetMeta updateLastGridDataSetMeta() {
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
lastGridDataSetMeta = null;
}
log.info("更新最新的数据元信息: {}", lastGridDataSetMeta);
return lastGridDataSetMeta;
}
/** /**
* (time, lev, lat, lon) * (time, lev, lat, lon)
@ -97,12 +139,19 @@ public class GfsDataFetcher extends BaseTableOperation {
* @return {@link Array} * @return {@link Array}
* @throws Exception * @throws Exception
*/ */
public Array getProfile(String dataSetId, String variable, int iTime, int iLat, int iLon) throws Exception { public Array getProfileByVariableAndPressure(String variableName, OffsetDateTime targetTime, double latitude, double longitude) throws Exception {
int[] origin = new int[] {iTime, 0, iLat, iLon}; lastGridDataSetMeta = getLastGridDataSetMeta();
int[] shape = new int[] {1, tableConfig.levSize, 1, 1}; GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(lastGridDataSetMeta);
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), int iTime = getTargetTimeIndex(targetTime);
origin, shape); int iLat= getLatitudeIndex(latitude);
Array array = gridDataSet.getVariable(variable).toArray(); int iLon = getLongitudeIndex(longitude);
int[] origin = new int[]{iTime, 0, iLat, iLon};
int[] shape = new int[]{1,tableConfig.levSize, 1, 1};
// TODO 2024/6/17:
fetcher.setVariablesToGet(Collections.singletonList(variableName));
fetcher.setOriginShape(origin, shape);
GridDataSet gridDataSet = fetcher.fetch();
Array array = gridDataSet.getVariable(variableName).toArray();
return array; return array;
} }
@ -141,4 +190,107 @@ public class GfsDataFetcher extends BaseTableOperation {
int[] shape = new int[] {iTImes.length, 1, 1, 1}; int[] shape = new int[] {iTImes.length, 1, 1, 1};
return queryByTableStore(dataSetId, variableList, origin, shape); return queryByTableStore(dataSetId, variableList, origin, shape);
} }
/**
* 4
* @param variableNames
* @param timeRange
* @param levRange
* @param latRange
* @param lonRange
* @return
* @throws Exception
*/
public GridDataSet queryDataset4D(List<String> variableNames, ValueRange<OffsetDateTime> timeRange, ValueRange<Integer> levRange, ValueRange<Double> latRange, ValueRange<Double> lonRange) throws Exception {
lastGridDataSetMeta = getLastGridDataSetMeta();
GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(lastGridDataSetMeta);
Range iTimeRange = new Range(getTargetTimeIndex(timeRange.getStart()),getTargetTimeIndex(timeRange.getEnd()) + 1);
Range iLevRage = new Range(getLevelIndex(levRange.getStart()), getLevelIndex(levRange.getEnd()) + 1);
Range iLatRange = new Range(getLatitudeIndex(latRange.getStart()),getLatitudeIndex(latRange.getEnd()) + 1);
Range iLonRange = new Range(getLongitudeIndex(lonRange.getStart()),getLongitudeIndex(lonRange.getEnd()) + 1);
int[] origin = new int[]{iTimeRange.getStart(), iLevRage.getStart(), iLatRange.getStart(), iLonRange.getStart()};
int[] shape = new int[]{iTimeRange.getSize(), iLevRage.getSize(), iLatRange.getSize(), iLonRange.getSize()};
// TODO 2024/6/17:
fetcher.setVariablesToGet(variableNames);
fetcher.setOriginShape(origin, shape);
GridDataSet gridDataSet = fetcher.fetch();
return gridDataSet;
}
private int getTargetTimeIndex(OffsetDateTime targetTime) {
lastGridDataSetMeta = getLastGridDataSetMeta();
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
if (targetTime.isBefore(refTime)) {
throw new AppException(ErrorCode.QUERY_TIME_ERROR);
}
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
String targetHour = String.valueOf(getForecastHourFromTargetTime(targetTime, refTime));
if (!forecastHours.contains(targetHour)) {
throw new AppException(ErrorCode.QUERY_TIME_ERROR);
}
int iTime = forecastHours.indexOf(targetHour);
return iTime;
}
public int getLongitudeIndex(double lon) {
double[] lonList = tableConfig.getLonList();
if (lonList.length == 0) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "经度列表为空, 无法查找索引");
}
double minLon = lonList[0];
double maxLon = lonList[lonList.length - 1];
if (minLon > lon || lon > maxLon) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "经度不在有效范围内: " + minLon + " ~ " + maxLon);
}
int nearestIndex = NdArrayUtils.findNearestIndex(lonList, lon);
if (nearestIndex == -1) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "找不到" + lon + "的经度索引");
}
return nearestIndex;
}
public int getLatitudeIndex(double lat) {
double[] latList = tableConfig.getLatList();
if (latList.length == 0) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "纬度列表为空, 无法查找索引");
}
double minLat = latList[0];
double maxLat = latList[latList.length - 1];
if (minLat > lat || lat > maxLat) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "纬度不在有效范围内: " + minLat + " ~ " + maxLat);
}
int nearestIndex = NdArrayUtils.findNearestIndex(latList, lat);
if (nearestIndex == -1) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + lat + "的纬度索引");
}
return nearestIndex;
}
private int getPressureIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfig.getPressureList()).boxed().collect(Collectors.toList());
if (CollectionUtils.isEmpty(levList)) {
throw new AppException(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空, 无法查找索引");
}
int index = levList.indexOf(pressureLevel);
if (index == -1) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + pressureLevel + "的气压高度索引");
}
return index;
}
public int getLevelIndex(int level) {
if (GfsLevelsEnum.getByCode(level) == GfsLevelsEnum.SURFACE) {
return 0; // NOTE 2024/5/20: 0 表示地面风速对应10m高度其他变量对应2m高度
} else {
return getPressureIndex(level);
}
}
private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) {
return (int) Duration.between(refTime, targetTime).toHours();
}
} }

@ -42,7 +42,8 @@ import java.util.concurrent.Executors;
public class GfsDataImport extends BaseTableOperation { public class GfsDataImport extends BaseTableOperation {
@Resource @Resource
TableConfig tableConfig; TableConfig tableConfig;
@Resource
GfsDataFetcher gfsDataFetcher;
@Getter @Getter
boolean importing = false; boolean importing = false;
private final ExecutorService executorService = Executors.newFixedThreadPool(5); private final ExecutorService executorService = Executors.newFixedThreadPool(5);
@ -97,12 +98,12 @@ public class GfsDataImport extends BaseTableOperation {
public List<ImportResult> importData(OffsetDateTime refTime) throws Exception { public List<ImportResult> importData(OffsetDateTime refTime) throws Exception {
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime); List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) { if (CollectionUtils.isEmpty(fileList)) {
throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES); throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES);
} }
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING)); String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfig.variableList); List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize}; int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
@ -140,6 +141,8 @@ public class GfsDataImport extends BaseTableOperation {
meta.setForecastHours(forecastHours); meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli()); meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta); putMeta(meta);
gfsDataFetcher.lastGridDataSetMeta = meta;
log.info("[tablestore]: 更新最新的数据元信息: {}", meta);
return finishedList; return finishedList;
} }

@ -0,0 +1,35 @@
package com.htfp.weather.griddata.utils;
import lombok.Getter;
import java.util.Collections;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/6/17 17:28
* @Description :
*/
@Getter
public class ValueRange<T extends Comparable<T>>{
T start;
T end; // include
public ValueRange(T start, T end) {
this.start = start;
this.end = end;
}
public ValueRange(List<T> list) {
this.start = Collections.min(list);
this.end = Collections.max(list);
}
@Override
public String toString() {
return "[" + start +
"," + end +
']';
}
}

@ -1,5 +1,7 @@
package com.htfp.weather.info; package com.htfp.weather.info;
import java.time.format.DateTimeFormatter;
/** /**
* @Author : shiyi * @Author : shiyi
* @Date : 2024/5/8 12:02 * @Date : 2024/5/8 12:02
@ -10,5 +12,6 @@ public class Constant {
public static final String DATA_FOLDER_STRING = "'UTC-'yyyyMMdd.HH"; public static final String DATA_FOLDER_STRING = "'UTC-'yyyyMMdd.HH";
public static final String UTC_TIME_STRING = "'UTC-'yyyyMMdd.HH"; public static final String UTC_TIME_STRING = "'UTC-'yyyyMMdd.HH";
public static final String BJT_TIME_STRING = "'BJT-'yyyyMMdd.HH"; public static final String BJT_TIME_STRING = "'BJT-'yyyyMMdd.HH";
public static final String API_TIME_STRING = "yyyy-MM-dd'T'HH:mmxxx"; public static final String API_TIME_STRING = "yyyy-MM-dd'T'HH:mm:ssxxx";
public static final DateTimeFormatter API_TIME_FORMATTER = DateTimeFormatter.ISO_OFFSET_DATE_TIME;
} }

@ -13,7 +13,7 @@ public enum GfsDownloadVariableEnum {
CLOUD("TCDC", "云量"), CLOUD("TCDC", "云量"),
RELATIVE_HUMIDITY("RH", "相对湿度"), RELATIVE_HUMIDITY("RH", "相对湿度"),
VERTICAL_VELOCITY("DZDT", "垂直速度"), VERTICAL_VELOCITY("DZDT", "垂直速度"),
PRECIPITATION("PRATE", "降水率(mm)"),
; ;
private final String code; private final String code;
private final String info; private final String info;

@ -1,8 +1,10 @@
package com.htfp.weather.schedule; package com.htfp.weather.schedule;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.htfp.weather.download.FileInfo; import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig; import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader; import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.GfsDataImport; import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult; import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.info.Constant; import com.htfp.weather.info.Constant;
@ -41,6 +43,8 @@ public class GridDataProcessor {
@Resource @Resource
GfsDownloader gfsDownloader; GfsDownloader gfsDownloader;
@Resource @Resource
GfsDataFetcher gfsDataFetcher;
@Resource
GfsDataConfig gfsDataConfig; GfsDataConfig gfsDataConfig;
@Resource @Resource
ErrorReportService errorReportService; ErrorReportService errorReportService;
@ -115,17 +119,19 @@ public class GridDataProcessor {
return; return;
} }
OffsetDateTime now = OffsetDateTime.now(); OffsetDateTime now = OffsetDateTime.now();
for (File subDir : dataDir.listFiles()) { File[] dirs = dataDir.listFiles();
for (File subDir : dirs) {
OffsetDateTime refTime = LocalDateTime.parse(subDir.getName(), DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)).atOffset(ZoneOffset.UTC); OffsetDateTime refTime = LocalDateTime.parse(subDir.getName(), DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)).atOffset(ZoneOffset.UTC);
if (refTime.isBefore(now.minusDays(3))) { if (refTime.isBefore(now.minusDays(3))) {
try { try {
FileUtils.deleteDirectory(subDir); FileUtils.deleteDirectory(subDir);
log.info("[schedule-end] 删除过期数据文件夹成功: {}", subDir.getName()); log.info("[schedule] 删除过期数据文件夹成功: {}", subDir.getName());
} catch (IOException e) { } catch (IOException e) {
log.info("[schedule-end] 删除过期数据文件夹失败: {}", subDir.getName()); log.info("[schedule] 删除过期数据文件夹失败: {}", subDir.getName());
} }
} }
} }
log.info("[schedule-end] 过期数据清理结束");
} }
private void downloadFailedReport(String refTime, List<FileInfo> finishedList, List<FileInfo> failedList) { private void downloadFailedReport(String refTime, List<FileInfo> finishedList, List<FileInfo> failedList) {

@ -67,7 +67,7 @@ public class ConfigController {
/** /**
* tablestore * tablestore
*/ */
@RequestMapping("/updateDatabaseConfig") @PostMapping("/updateDatabaseConfig")
public Result updateDatabaseConfig(@RequestBody TableConfigUpdate request) { public Result updateDatabaseConfig(@RequestBody TableConfigUpdate request) {
String secret = request.getSecret(); String secret = request.getSecret();
TableConfig updateConfig = request.getConfig(); TableConfig updateConfig = request.getConfig();
@ -95,7 +95,7 @@ public class ConfigController {
/** /**
* GFS * GFS
*/ */
@RequestMapping("/updateDataSourceConfig") @PostMapping("/updateDataSourceConfig")
public Result updateDataSourceConfig(@Validated @RequestBody GfsConfigUpdate request) { public Result updateDataSourceConfig(@Validated @RequestBody GfsConfigUpdate request) {
String secret = request.getSecret(); String secret = request.getSecret();
GfsDataConfig updateConfig = request.getConfig(); GfsDataConfig updateConfig = request.getConfig();
@ -115,7 +115,7 @@ public class ConfigController {
/** /**
* 24 * 24
*/ */
@RequestMapping("/downloadAllFiles") @PostMapping("/downloadAllFiles")
public Result downloadAllFiles(@RequestBody Map<String, String> params) { public Result downloadAllFiles(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);
@ -142,11 +142,11 @@ public class ConfigController {
/** /**
* *
*/ */
@RequestMapping("/downloadSingleFile") @PostMapping("/downloadSingleFile")
public Result downloadSingleFile(@RequestBody Map<String, String> params) { public Result downloadSingleFile(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);
OffsetDateTime offsetDateTime = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); OffsetDateTime offsetDateTime = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)); String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING));
try { try {
gfsDownloader.iniTimeSetting(); gfsDownloader.iniTimeSetting();
@ -170,11 +170,11 @@ public class ConfigController {
/** /**
* *
*/ */
@RequestMapping("/importToTablestoreByReferenceTime") @PostMapping("/importToTablestoreByReferenceTime")
public Result importData(@RequestBody Map<String, String> params) { public Result importData(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); OffsetDateTime time = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
try { try {
List<ImportResult> finishedList = gfsDataImport.importData(time); List<ImportResult> finishedList = gfsDataImport.importData(time);
List<ImportResult> successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList()); List<ImportResult> successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
@ -194,14 +194,22 @@ public class ConfigController {
} }
} }
@PostMapping("queryAllDataDir")
public Result queryAllDataDir(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
File dataRoot = new File(gfsDataConfig.getSaveRoot());
List<String> subDirList = fileService.getSubDirList(dataRoot);
return Result.success(subDirList);
}
/** /**
* *
*/ */
@RequestMapping("/queryAllFileByReferenceTime") @PostMapping("/queryAllFileByReferenceTime")
public Result queryAllFileByReferenceTime(@RequestBody Map<String, String> params) { public Result queryAllFileByReferenceTime(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); OffsetDateTime time = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr); File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr);
@ -213,11 +221,11 @@ public class ConfigController {
/** /**
* *
*/ */
@RequestMapping("/deleteDirByReferenceTime") @PostMapping("/deleteDirByReferenceTime")
public Result deleteDirByReferenceTime(@RequestBody Map<String, String> params) { public Result deleteDirByReferenceTime(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); OffsetDateTime time = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr); File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr);
if (gfsDataImport.isImporting()) { if (gfsDataImport.isImporting()) {
@ -234,7 +242,7 @@ public class ConfigController {
/** /**
* *
*/ */
@RequestMapping("/deleteTargetFile") @PostMapping("/deleteTargetFile")
public Result deleteTargetFile(@RequestBody Map<String, String> params) { public Result deleteTargetFile(@RequestBody Map<String, String> params) {
String secret = params.get("secret"); String secret = params.get("secret");
validSecret(secret); validSecret(secret);

@ -10,6 +10,7 @@ import com.htfp.weather.web.service.surfaceapi.CaiYunServiceImpl;
import com.htfp.weather.web.service.surfaceapi.HeFengServiceImpl; import com.htfp.weather.web.service.surfaceapi.HeFengServiceImpl;
import com.htfp.weather.web.service.IDataService; import com.htfp.weather.web.service.IDataService;
import com.htfp.weather.web.service.surfaceapi.ISurfaceDataService; import com.htfp.weather.web.service.surfaceapi.ISurfaceDataService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
@ -22,19 +23,21 @@ import java.util.List;
* @Description : * @Description :
*/ */
@CrossOrigin @CrossOrigin
@RestController @RestController @Slf4j
@RequestMapping("/htfp/weather/surface/") @RequestMapping("/htfp/weather/surface/")
public class SurfaceWeatherController { public class SurfaceWeatherController {
@Resource(name = "tablestore-gfs") @Resource(name = "hefeng")
IDataService dataService;
@Resource(name = "caiyun")
ISurfaceDataService surfaceDataService; ISurfaceDataService surfaceDataService;
@PostMapping("/now") @PostMapping("/now")
public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception { public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude(); double lat = position2D.getLatitude();
double lon = position2D.getLongitude(); double lon = position2D.getLongitude();
log.info("[data-server] 地面实时气象信息查询 start: param={}", position2D);
NowWeatherStatus nowWeatherStatus = surfaceDataService.getNowSurfaceWeatherStatus(lat, lon); NowWeatherStatus nowWeatherStatus = surfaceDataService.getNowSurfaceWeatherStatus(lat, lon);
nowWeatherStatus.setLatitude(lat);
nowWeatherStatus.setLongitude(lon);
log.info("[data-server] 地面实时气象信息查询 end");
return Result.success(nowWeatherStatus); return Result.success(nowWeatherStatus);
} }
@ -42,7 +45,11 @@ public class SurfaceWeatherController {
public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception { public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude(); double lat = position2D.getLatitude();
double lon = position2D.getLongitude(); double lon = position2D.getLongitude();
log.info("[data-server] 地面24小时预报结果查询 start: param={}", position2D);
TimeSeriesDataset forecastSeries = surfaceDataService.getForecastSeries(lat, lon); TimeSeriesDataset forecastSeries = surfaceDataService.getForecastSeries(lat, lon);
forecastSeries.setLatitude(lat);
forecastSeries.setLongitude(lon);
log.info("[data-server] 地面24小时预报结果查询");
return Result.success(forecastSeries); return Result.success(forecastSeries);
} }
@ -50,7 +57,9 @@ public class SurfaceWeatherController {
public Result warning(@Validated @RequestBody Position2D position2D) throws Exception { public Result warning(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude(); double lat = position2D.getLatitude();
double lon = position2D.getLongitude(); double lon = position2D.getLongitude();
log.info("[data-server] 地面气象预警信息查询 start: param={}", position2D);
List<SurfaceWeatherWarning> warning = surfaceDataService.getSurfaceWarning(lat, lon); List<SurfaceWeatherWarning> warning = surfaceDataService.getSurfaceWarning(lat, lon);
log.info("[data-server] 地面气象预警信息查询 end");
return Result.success(warning); return Result.success(warning);
} }

@ -1,11 +1,13 @@
package com.htfp.weather.web.controller; package com.htfp.weather.web.controller;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode; import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.*; import com.htfp.weather.web.pojo.request.*;
import com.htfp.weather.web.pojo.response.*; import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.service.GfsDataServiceImpl; import com.htfp.weather.web.service.GfsDataServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
@ -20,55 +22,21 @@ import java.util.List;
* @Description : * @Description :
*/ */
@CrossOrigin @CrossOrigin
@RestController @RestController @Slf4j
@RequestMapping("/htfp/weather/upper/") @RequestMapping("/htfp/weather/upper/")
public class UpperWeatherController { public class UpperWeatherController {
@Resource(name = "tablestore-gfs") @Resource(name = "tablestore-gfs")
GfsDataServiceImpl gfsDataService; GfsDataServiceImpl gfsDataService;
@RequestMapping("/profileByPressure")
public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
ProfileResponse profileResponse = gfsDataService.getProfile(utcDateTime, variableName, latitude, longitude);
return Result.success(profileResponse);
}
@PostMapping("/queryForecastTimeSeries")
public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
TimeSeriesDataset forecastSeries = gfsDataService.getForecastSeries(latitude, longitude, level);
return Result.success(forecastSeries);
}
@PostMapping("/plane")
public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) {
planeRequest.valid();
String variableName = planeRequest.getVariableName();
OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
int level = planeRequest.getLevel();
double minLat = planeRequest.getMinLat();
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
return Result.success(forecastSeries);
}
@PostMapping("/queryUpperNowWeather") @PostMapping("/queryUpperNowWeather")
public Result queryUpperNowWeather(@Validated @RequestBody Position3D position3D) { public Result queryUpperNowWeather(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude(); double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude(); double longitude = position3D.getLongitude();
int level = position3D.getLevel(); int level = position3D.getLevel();
NowWeatherStatus nowWeather = gfsDataService.getNowWeather(latitude, longitude, level); log.info("[data-server] 高空实时气象信息查询 start: param={}", position3D);
NowWeatherStatus nowWeather = gfsDataService.getNowWeather(level, latitude, longitude);
log.info("[data-server] 高空实时气象信息查询 end");
return Result.success(nowWeather); return Result.success(nowWeather);
} }
@ -87,10 +55,23 @@ public class UpperWeatherController {
} }
} }
int level = multiPointsRequest.getLevel().intValue(); int level = multiPointsRequest.getLevel().intValue();
List<NowWeatherStatus> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level); log.info("[data-server] 高空航点实时气象信息查询 start: param={}", multiPointsRequest);
List<NowWeatherStatus> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(level, latitudeList, longitudeList);
log.info("[data-server] 高空航点实时气象信息查询 end");
return Result.success(nowWeatherByMultiPoint); return Result.success(nowWeatherByMultiPoint);
} }
@PostMapping("/queryForecastTimeSeries24Hour")
public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
log.info("[data-server] 高空24小时预报结果查询 start: param={}", position3D);
TimeSeriesDataset forecastSeries = gfsDataService.getForecastSeries(level, latitude, longitude);
log.info("[data-server] 高空24小时预报结果查询 end");
return Result.success(forecastSeries);
}
@PostMapping("/queryForecastTimeSeriesInMultiPoints") @PostMapping("/queryForecastTimeSeriesInMultiPoints")
public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) { public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>(); List<Double> latitudeList = new ArrayList<>();
@ -106,8 +87,41 @@ public class UpperWeatherController {
} }
} }
int level = multiPointsRequest.getLevel().intValue(); int level = multiPointsRequest.getLevel().intValue();
log.info("[data-server] 高空航点24小时预报结果查询 start: param={}", multiPointsRequest);
List<TimeSeriesDataset> nowWeatherByMultiPoint = gfsDataService.getForecastSeriesByMultiPoint(latitudeList, longitudeList, level); List<TimeSeriesDataset> nowWeatherByMultiPoint = gfsDataService.getForecastSeriesByMultiPoint(latitudeList, longitudeList, level);
log.info("[data-server] 高空航点24小时预报结果查询 end");
return Result.success(nowWeatherByMultiPoint); return Result.success(nowWeatherByMultiPoint);
} }
@RequestMapping("/queryProfileByVariableAndPressure")
public Result queryProfileByVariableAndPressure(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
log.info("[data-server] 高度廓线查询 start: param={}", profileRequest);
ProfileResponse profileResponse = gfsDataService.getProfileByVariableAndPressure(variableName, utcDateTime, latitude, longitude);
log.info("[data-server] 高度廓线查询 end");
return Result.success(profileResponse);
}
@PostMapping("/plane")
public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) {
planeRequest.valid();
String variableName = planeRequest.getVariableName();
OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
int level = planeRequest.getLevel();
double minLat = planeRequest.getMinLat();
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
log.info("[data-server] 平面网格数据查询 start: param={}", planeRequest);
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
log.info("[data-server] 平面网格数据查询 end");
return Result.success(forecastSeries);
}
} }

@ -34,7 +34,8 @@ public enum ErrorCode {
LEVEL_INDEX_ERROR(6004, "高度坐标索引错误"), LEVEL_INDEX_ERROR(6004, "高度坐标索引错误"),
INDEX_SIZE_ERROR(6005, "坐标长度不一致"), INDEX_SIZE_ERROR(6005, "坐标长度不一致"),
NO_SUCH_VARIABLE(6006, "找不到变量名映射"), NO_SUCH_VARIABLE(6006, "找不到变量名映射"),
RESPONSE_DATA_BUILD_ERROR(6007, "查询数据结果构建错误"), TABLESTORE_QUERY_ERROR(6007, "数据库查询错误"),
RESPONSE_DATA_BUILD_ERROR(6008, "查询数据结果构建错误"),
; ;
private final int code; private final int code;
private final String msg; private final String msg;

@ -0,0 +1,18 @@
package com.htfp.weather.web.pojo.request;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/6/3 19:21
* @Description :
*/
@Data
public class MultiPointsRequest {
List<Position2D> pointList;
@NotNull(message = "高度层次 (level) 不能为空")
Integer level;
}

@ -21,7 +21,7 @@ import javax.validation.constraints.NotNull;
@Data @Data
public class PlaneRequest { public class PlaneRequest {
@NotNull(message = "时间不能为空") @NotNull(message = "时间不能为空")
@DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx") @DateTimeStr(message = "时间格式错误")
String time; String time;
@NotNull(message = "变量名不能为空") @NotNull(message = "变量名不能为空")

@ -16,7 +16,7 @@ import javax.validation.constraints.NotNull;
@Data @Data
public class ProfileRequest { public class ProfileRequest {
@NotNull(message = "时间不能为空") @NotNull(message = "时间不能为空")
@DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx") @DateTimeStr(message = "时间格式错误")
String time; String time;
@NotNull(message = "变量名不能为空") @NotNull(message = "变量名不能为空")

@ -1,6 +1,7 @@
package com.htfp.weather.web.pojo.response; package com.htfp.weather.web.pojo.response;
import lombok.Data; import lombok.Data;
import lombok.Getter;
/** /**
* @Author : shiyi * @Author : shiyi
@ -11,6 +12,8 @@ import lombok.Data;
public class ProfileResponse { public class ProfileResponse {
int[] pressureLevels; int[] pressureLevels;
// int[ pressureHeight; // int[ pressureHeight;
double latitude;
double longitude;
float[] values; float[] values;
public ProfileResponse(int[] pressureLevels, float[] values) { public ProfileResponse(int[] pressureLevels, float[] values) {

@ -40,6 +40,23 @@ public class FileService {
} }
return fileInfoList; return fileInfoList;
} }
public List<String> getSubDirList(File dataRoot){
if (!dataRoot.exists() || !dataRoot.isDirectory()) {
throw new AppException(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS);
}
List<String> subDirList = new ArrayList<>();
File[] files = dataRoot.listFiles();
if (files == null || files.length == 0) {
return null;
}
for (File file : files) {
if (file.isDirectory()) {
subDirList.add(file.getName());
}
}
return subDirList;
}
public void uploadFile(String filePath){ public void uploadFile(String filePath){
// 文件上传逻辑 // 文件上传逻辑
} }

@ -8,11 +8,10 @@ import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum; import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig; import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataFetcher; import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.QueryMeta; import com.htfp.weather.griddata.utils.ValueRange;
import com.htfp.weather.info.Constant; import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsLevelsEnum; import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode; import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.NowWeatherStatus; import com.htfp.weather.web.pojo.response.NowWeatherStatus;
@ -33,7 +32,6 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
/** /**
* @Author : shiyi * @Author : shiyi
@ -43,90 +41,12 @@ import java.util.stream.Collectors;
@Service("tablestore-gfs") @Service("tablestore-gfs")
@Slf4j @Slf4j
public class GfsDataServiceImpl implements IDataService { public class GfsDataServiceImpl implements IDataService {
private static final int SURFACE_LEVEL = 9999;
@Resource @Resource
GfsDataFetcher gfsDataFetcher; GfsDataFetcher gfsDataFetcher;
@Resource @Resource
QueryMeta queryMeta;
@Resource
TableConfig tableConfig; TableConfig tableConfig;
/**
* 线
*
* @param targetTime UTC
* @param variableName
* @param latitude
* @param longitude
* @return {@link ProfileResponse}
*/
public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) {
String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
if (targetVariable == null) {
throw new AppException(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
}
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
int[] pressureLevels = tableConfig.getPressureList();
Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon);
float[] values = (float[]) array.copyTo1DJavaArray();
return new ProfileResponse(pressureLevels, values);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 24
*
* @param latitude
* @param longitude
* @param level 9999
* @return @{@link TimeSeriesDataset}
*/
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
Map<String, List> params = getFutureTimeIndexList(lastGridDataSetMeta);
List<Integer> iTimeList = params.get("iTimeList");
List<String> timeList = params.get("timeList");
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray();
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
TimeSeriesDataset timeSeriesDataset = buildTimeSeriesDatasetInTargetPoint(timeList, gridDataSet, 0, 0, 0, levelFlag);
timeSeriesDataset.setLatitude(latitude);
timeSeriesDataset.setLongitude(longitude);
return timeSeriesDataset;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/** /**
* *
@ -136,26 +56,20 @@ public class GfsDataServiceImpl implements IDataService {
* @param level 9999 * @param level 9999
* @return @{@link NowWeatherStatus} * @return @{@link NowWeatherStatus}
*/ */
public NowWeatherStatus getNowWeather(double latitude, double longitude, int level) { public NowWeatherStatus getNowWeather(int level, double latitude, double longitude) {
GridDataSetMeta lastGridDataSetMeta = null; OffsetDateTime nowTime = OffsetDateTime.now();
try { ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Collections.singletonList(nowTime));
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); ValueRange<Integer> levRange = new ValueRange<>(Collections.singletonList(level));
} catch (Exception e) { ValueRange<Double> latRange = new ValueRange<>(Collections.singletonList(latitude));
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage()); ValueRange<Double> lonRange = new ValueRange<>(Collections.singletonList(longitude));
}
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int iLev = getLevelIndex(level);
List<String> variableNames = getVariableNamesInDatabase(level); List<String> variableNames = getVariableNamesInDatabase(level);
try { try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); GridDataSet gridDataSet = gfsDataFetcher.queryDataset4D(
GridDataSet gridDataSet = gfsDataFetcher.getSinglePoint(gridDataSetId, variableNames, iTime, iLev, iLat, iLon); variableNames,timeRange, levRange, latRange, lonRange
);
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level); final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
// 每个变量都是0维数组 // 每个变量都是0维数组
NowWeatherStatus nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, 0, 0, levelFlag); NowWeatherStatus nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, level, latitude, longitude, levelFlag);
nowWeatherStatus.setLatitude(latitude); nowWeatherStatus.setLatitude(latitude);
nowWeatherStatus.setLongitude(longitude); nowWeatherStatus.setLongitude(longitude);
return nowWeatherStatus; return nowWeatherStatus;
@ -171,51 +85,58 @@ public class GfsDataServiceImpl implements IDataService {
* @param level * @param level
* @return * @return
*/ */
public List<NowWeatherStatus> getNowWeatherByMultiPoint(List<Double> latitude, List<Double> longitude, int level) { public List<NowWeatherStatus> getNowWeatherByMultiPoint(int level, List<Double> latitude, List<Double> longitude) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage());
}
if (latitude.size() != longitude.size()) { if (latitude.size() != longitude.size()) {
throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致"); throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致");
} }
int pointSize = latitude.size(); int pointSize = latitude.size();
List<String> variableNames = getVariableNamesInDatabase(level); List<String> variableNames = getVariableNamesInDatabase(level);
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta); OffsetDateTime nowTime = OffsetDateTime.now();
List<Integer> latIndex = latitude.stream().map(this::getLatitudeIndex).collect(Collectors.toList()); ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Collections.singletonList(nowTime));
List<Integer> lonIndex = longitude.stream().map(this::getLongitudeIndex).collect(Collectors.toList()); ValueRange<Integer> levRange = new ValueRange<>(Collections.singletonList(level));
int iMinLat = Collections.min(latIndex).intValue(); ValueRange<Double> latRange = new ValueRange<>(latitude);
int iMaxLat = Collections.max(latIndex).intValue(); ValueRange<Double> lonRange = new ValueRange<>(longitude);
int iMinLon = Collections.min(lonIndex).intValue();
int iMaxLon = Collections.max(lonIndex).intValue();
int iLev = getLevelIndex(level);
int latLength = iMaxLat - iMinLat + 1;
int lonLength = iMaxLon - iMinLon + 1;
// int levLength = iMaxLev - iMinLev + 1;
try { try {
GridDataSet gridDataSet = gfsDataFetcher.queryDataset4D(
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); variableNames,timeRange, levRange, latRange, lonRange
int[] origin = new int[]{iTime, iLev, 0, 0}; );
int[] shape = new int[]{1, 1, tableConfig.getLatSize(), tableConfig.getLonSize()};
GridDataSet gridDataSet = gfsDataFetcher.queryByTableStore(gridDataSetId, variableNames,
origin, shape);
List<NowWeatherStatus> nowWeatherStatusList = new ArrayList<>(); List<NowWeatherStatus> nowWeatherStatusList = new ArrayList<>();
for (int i = 0; i < pointSize; i++) { for (int i = 0; i < pointSize; i++) {
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); NowWeatherStatus nowWeatherStatus;
// NOTE 2024/6/11: build中传入的索引和数据库的查询索引意义不同 // NOTE 2024/6/11: build中传入的索引和数据库的查询索引意义不同
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level); final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, latIndex.get(i), lonIndex.get(i), levelFlag); nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, level, latitude.get(i), longitude.get(i), levelFlag);
nowWeatherStatus.setLatitude(latitude.get(i));
nowWeatherStatus.setLongitude(longitude.get(i));
nowWeatherStatusList.add(nowWeatherStatus); nowWeatherStatusList.add(nowWeatherStatus);
} }
return nowWeatherStatusList; return nowWeatherStatusList;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 24
*
* @param latitude
* @param longitude
* @param level 9999
* @return @{@link TimeSeriesDataset}
*/
public TimeSeriesDataset getForecastSeries(int level, double latitude, double longitude) {
List<String> variableNames = getVariableNamesInDatabase(level);
OffsetDateTime nowTime = OffsetDateTime.now();
ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Arrays.asList(nowTime, nowTime.plusHours(24)));
ValueRange<Integer> levRange = new ValueRange<>(Collections.singletonList(level));
ValueRange<Double> latRange = new ValueRange<>(Collections.singletonList(latitude));
ValueRange<Double> lonRange = new ValueRange<>(Collections.singletonList(longitude));
try {
GridDataSet gridDataSet = gfsDataFetcher.queryDataset4D(
variableNames, timeRange, levRange, latRange, lonRange
);
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
return buildTimeSeriesDatasetInTargetPoint(gridDataSet, level, latitude, longitude, levelFlag);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -223,79 +144,71 @@ public class GfsDataServiceImpl implements IDataService {
/** /**
* 24 * 24
* @param latitude * @param latitudeList
* @param longitude * @param longitudeList
* @param level * @param level
* @return * @return
*/ */
public List<TimeSeriesDataset> getForecastSeriesByMultiPoint(List<Double> latitude, List<Double> longitude, int level) { public List<TimeSeriesDataset> getForecastSeriesByMultiPoint(List<Double> latitudeList, List<Double> longitudeList, int level) {
GridDataSetMeta lastGridDataSetMeta = null; if (latitudeList.size() != longitudeList.size()) {
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage());
}
if (latitude.size() != longitude.size()) {
throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致"); throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致");
} }
int pointSize = latitude.size(); int pointSize = latitudeList.size();
List<String> variableNames = getVariableNamesInDatabase(level); List<String> variableNames = getVariableNamesInDatabase(level);
Map<String, List> params = getFutureTimeIndexList(lastGridDataSetMeta); OffsetDateTime nowTime = OffsetDateTime.now();
List<Integer> iTimeList = params.get("iTimeList"); ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Arrays.asList(nowTime, nowTime.plusHours(24)));
List<String> timeList = params.get("timeList"); ValueRange<Integer> levRange = new ValueRange<>(Collections.singletonList(level));
int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray(); ValueRange<Double> latRange = new ValueRange<>(latitudeList);
List<Integer> latIndex = latitude.stream().map(this::getLatitudeIndex).collect(Collectors.toList()); ValueRange<Double> lonRange = new ValueRange<>(longitudeList);
List<Integer> lonIndex = longitude.stream().map(this::getLongitudeIndex).collect(Collectors.toList());
int iLev = getLevelIndex(level);
try { try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); GridDataSet gridDataSet = gfsDataFetcher.queryDataset4D(
variableNames, timeRange, levRange, latRange, lonRange
int[] origin = new int[]{iTimes[0], iLev, 0, 0}; );
// TODO 2024/6/12: 全域查询量太大,可以考虑只在航线所在区域查询,提升查询效率
int[] shape = new int[]{iTimes.length, 1, tableConfig.getLatSize(), tableConfig.getLonSize()};
GridDataSet gridDataSet = gfsDataFetcher.queryByTableStore(gridDataSetId, variableNames,
origin, shape);
List<TimeSeriesDataset> timeSeriesDatasetList = new ArrayList<>(); List<TimeSeriesDataset> timeSeriesDatasetList = new ArrayList<>();
for (int i = 0; i < pointSize; i++) { for (int i = 0; i < pointSize; i++) {
// NOTE 2024/6/11: build中传入的索引和数据库的查询索引意义不同
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level); final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
TimeSeriesDataset timeSeriesDataset = buildTimeSeriesDatasetInTargetPoint(timeList, gridDataSet, 0, latIndex.get(i), lonIndex.get(i), levelFlag); TimeSeriesDataset timeSeriesDataset = buildTimeSeriesDatasetInTargetPoint(gridDataSet, level, latitudeList.get(i), longitudeList.get(i), levelFlag);
timeSeriesDataset.setLatitude(latitude.get(i));
timeSeriesDataset.setLongitude(longitude.get(i));
timeSeriesDatasetList.add(timeSeriesDataset); timeSeriesDatasetList.add(timeSeriesDataset);
} }
return timeSeriesDatasetList; return timeSeriesDatasetList;
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
private int[] getMarginIndexByMultiPoint(List<Double> latitude, List<Double> longitude, List<Double> level) { /**
int minLat = Integer.MIN_VALUE; * 线
int maxLat = Integer.MAX_VALUE; *
int minLon = Integer.MIN_VALUE; * @param targetTime UTC
int maxLon = Integer.MAX_VALUE; * @param variableName
int minLev = Integer.MIN_VALUE; * @param latitude
int maxLev = Integer.MAX_VALUE; * @param longitude
* @return {@link ProfileResponse}
*/
int iMinLat = getLatitudeIndex(minLat); public ProfileResponse getProfileByVariableAndPressure(String variableName, OffsetDateTime targetTime, double latitude, double longitude) {
int iMaxLat = getLatitudeIndex(maxLat); String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
int iMinLon = getLongitudeIndex(minLon); if (targetVariable == null) {
int iMaxLon = getLongitudeIndex(maxLon); throw new AppException(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
}
try {
int[] pressureLevels = tableConfig.getPressureList();
Array array = gfsDataFetcher.getProfileByVariableAndPressure(targetVariable, targetTime, latitude, longitude);
float[] values = (float[]) array.copyTo1DJavaArray();
ProfileResponse profileResponse = new ProfileResponse(pressureLevels, values);
profileResponse.setLatitude(latitude);
profileResponse.setLongitude(longitude);
return profileResponse;
} catch (Exception e) {
throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage());
}
return new int[]{iMinLat, iMaxLat, iMinLon, iMaxLon};
} }
/** /**
* *
* *
* @param targetTime UTC * @param targetTime UTC
* @param variableName * @param variableName api
* @param level * @param level
* @param minLat * @param minLat
* @param maxLat * @param maxLat
@ -304,39 +217,28 @@ public class GfsDataServiceImpl implements IDataService {
* @return @{@link PlaneResponse} * @return @{@link PlaneResponse}
*/ */
public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) { public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) {
int iLev;
String targetVariable; String targetVariable;
if (level == SURFACE_LEVEL) { if (GfsLevelsEnum.SURFACE.equals(GfsLevelsEnum.getByCode(level))) {
iLev = 0; // NOTE 2024/5/20: 0 表示地面风速对应10m高度其他变量对应2m高度
targetVariable = GfsVariableHeightEnum.getGfsVariableName(variableName); targetVariable = GfsVariableHeightEnum.getGfsVariableName(variableName);
} else { } else {
iLev = getPressureIndex(level);
targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName); targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
} }
if (targetVariable == null) { if (targetVariable == null) {
throw new AppException(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在"); throw new AppException(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在");
} }
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppException(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
}
int iTime = getTargetTimeIndex(targetTime, lastGridDataSetMeta); List<String> variableNames = Collections.singletonList(targetVariable);
int iMinLat = getLatitudeIndex(minLat); ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Collections.singletonList(targetTime));
int iMaxLat = getLatitudeIndex(maxLat); ValueRange<Integer> levRange = new ValueRange<>(Collections.singletonList(level));
int iMinLon = getLongitudeIndex(minLon); ValueRange<Double> latRange = new ValueRange<>(minLat, maxLat);
int iMaxLon = getLongitudeIndex(maxLon); ValueRange<Double> lonRange = new ValueRange<>(minLon, maxLon);
int latLength = iMaxLat - iMinLat + 1;
int lonLength = iMaxLon - iMinLon + 1;
try { try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); Array array = gfsDataFetcher.queryDataset4D(
Array array = gfsDataFetcher.getPlane(gridDataSetId, targetVariable, iTime, iLev, iMinLat, iMinLon, latLength, lonLength); variableNames, timeRange, levRange, latRange, lonRange
).getVariable(targetVariable).toArray();
return buildPlane(array, iMinLat, iMinLon, latLength, lonLength); return buildPlane(array, minLat, maxLat, minLon, maxLon);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -346,7 +248,7 @@ public class GfsDataServiceImpl implements IDataService {
private List<String> getVariableNamesInDatabase(int level) { private List<String> getVariableNamesInDatabase(int level) {
List<String> variableNames = new ArrayList<>(); List<String> variableNames = new ArrayList<>();
if (level == SURFACE_LEVEL) { if (GfsLevelsEnum.SURFACE.equals(GfsLevelsEnum.getByCode(level))) {
for (String e : tableConfig.getVariableList()) { for (String e : tableConfig.getVariableList()) {
String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e); String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e);
if (gfsVariableName != null) { if (gfsVariableName != null) {
@ -364,7 +266,13 @@ public class GfsDataServiceImpl implements IDataService {
return variableNames; return variableNames;
} }
private PlaneResponse buildPlane(Array array, int iMinLat, int iMinLon, int latLength, int lonLength) { private PlaneResponse buildPlane(Array array, double minLat, double maxLat, double minLon, double maxLon) {
int iMinLat = gfsDataFetcher.getLatitudeIndex(minLat);
int iMaxLat = gfsDataFetcher.getLatitudeIndex(maxLat);
int iMinLon = gfsDataFetcher.getLongitudeIndex(minLon);
int iMaxLon = gfsDataFetcher.getLongitudeIndex(maxLon);
int latLength = iMaxLat - iMinLat + 1;
int lonLength = iMaxLon - iMinLon + 1;
PlaneResponse planeResponse = new PlaneResponse(); PlaneResponse planeResponse = new PlaneResponse();
float[][] values = (float[][]) array.reduce().copyToNDJavaArray(); float[][] values = (float[][]) array.reduce().copyToNDJavaArray();
planeResponse.setLatList(Arrays.copyOfRange(tableConfig.getLatList(), iMinLat, iMinLat + latLength)); planeResponse.setLatList(Arrays.copyOfRange(tableConfig.getLatList(), iMinLat, iMinLat + latLength));
@ -376,18 +284,23 @@ public class GfsDataServiceImpl implements IDataService {
/** /**
* *
* @param timeList-
* @param gridDataSet * @param gridDataSet
* @param iLevInGrid- * @param level
* @param iLatInGrid- * @param latitude
* @param iLonInGrid- * @param longitude
* @param levelFlag- * @param levelFlag-
* @return * @return
*/ */
private TimeSeriesDataset buildTimeSeriesDatasetInTargetPoint(List<String> timeList, GridDataSet gridDataSet, int iLevInGrid, int iLatInGrid, int iLonInGrid, GfsLevelsEnum levelFlag) {
private TimeSeriesDataset buildTimeSeriesDatasetInTargetPoint(GridDataSet gridDataSet, int level, double latitude, double longitude,GfsLevelsEnum levelFlag) {
TimeSeriesDataset result = new TimeSeriesDataset(); TimeSeriesDataset result = new TimeSeriesDataset();
result.setTime(timeList); result.setLatitude(latitude);
int tsize = timeList.size(); result.setLongitude(longitude);
// 获取全局索引
int levIndexGlobal = gfsDataFetcher.getLevelIndex(level);
int latIndexGlobal = gfsDataFetcher.getLatitudeIndex(latitude);
int lonIndexGlobal = gfsDataFetcher.getLongitudeIndex(longitude);
// 利用反射构建数据集 // 利用反射构建数据集
Class<? extends TimeSeriesDataset> aClass = result.getClass(); Class<? extends TimeSeriesDataset> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) { for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
@ -401,12 +314,17 @@ public class GfsDataServiceImpl implements IDataService {
if (variableNameInApi == null) { if (variableNameInApi == null) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE); throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
} }
Grid4D grid4D = entry.getValue();
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
int[] shape = grid4D.getShape();
int tSize = shape[0];
try { try {
Grid4D grid4D = entry.getValue();
String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1); String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1);
Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType()); Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType());
float[] storage = (float[]) grid4D.toArray().section(new int[]{0, iLevInGrid, iLatInGrid, iLonInGrid}, new int[]{tsize, 1, 1, 1}).copyTo1DJavaArray(); // TODO 2024/6/20: 这里的toArray操作在数据量较大的时候会比较吃内存和cpu
float[] storage = (float[]) grid4D.toArray().section(
new int[]{0, levIndexGlobal-origin[1], latIndexGlobal-origin[2], lonIndexGlobal-origin[3]}, new int[]{tSize, 1, 1, 1}
).copyTo1DJavaArray();
method.invoke(result, storage); method.invoke(result, storage);
} catch (ReflectiveOperationException e) { } catch (ReflectiveOperationException e) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e); throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e);
@ -414,23 +332,52 @@ public class GfsDataServiceImpl implements IDataService {
e.printStackTrace(); e.printStackTrace();
throw new AppException(ErrorCode.RESPONSE_DATA_BUILD_ERROR, e); throw new AppException(ErrorCode.RESPONSE_DATA_BUILD_ERROR, e);
} }
// 设置时间
if (result.getTime() == null) {
List<String> timeList = getResultTimeStringList(gridDataSet, tSize, origin[0]);
result.setTime(timeList);
}
} }
return result; return result;
} }
private List<String> getResultTimeStringList(GridDataSet gridDataSet, int timeSize, int iTimeStart) {
GridDataSetMeta meta = gridDataSet.getMeta();
List<String> forecastHours = meta.getForecastHours();
long milli = (long) meta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
List<String> timeList = new ArrayList<>();
for (int i = 0; i < timeSize; i++) {
int hour = Integer.parseInt(forecastHours.get(i + iTimeStart));
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(Constant.API_TIME_FORMATTER);
timeList.add(timeStr);
}
return timeList;
}
/** /**
* *
* @param gridDataSet * @param gridDataSet
* @param iLevInGrid- * @param level
* @param iLatInGrid- * @param latitude
* @param iLonInGrid- * @param longitude
* @param levelFlag- * @param levelFlag-
* @return * @return
*/ */
private NowWeatherStatus buildNowWeatherInTargetPoint(GridDataSet gridDataSet, int iLevInGrid, int iLatInGrid, int iLonInGrid, GfsLevelsEnum levelFlag) { private NowWeatherStatus buildNowWeatherInTargetPoint(GridDataSet gridDataSet, int level, double latitude, double longitude, GfsLevelsEnum levelFlag) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(Constant.API_TIME_FORMATTER);
NowWeatherStatus result = new NowWeatherStatus(); NowWeatherStatus result = new NowWeatherStatus();
result.setTime(timeStr); result.setTime(timeStr);
result.setLatitude(latitude);
result.setLongitude(longitude);
// 获取全局索引
int levIndexGlobal = gfsDataFetcher.getLevelIndex(level);
int latIndexGlobal = gfsDataFetcher.getLatitudeIndex(latitude);
int lonIndexGlobal = gfsDataFetcher.getLongitudeIndex(longitude);
// 利用反射构建数据集 // 利用反射构建数据集
Class<? extends NowWeatherStatus> aClass = result.getClass(); Class<? extends NowWeatherStatus> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) { for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
@ -445,10 +392,14 @@ public class GfsDataServiceImpl implements IDataService {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE); throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
} }
Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape()); Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape());
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
try { try {
String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1); String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1);
Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType()); Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType());
method.invoke(result, grid4D.toArray().getFloat(index.set(0, iLevInGrid, iLatInGrid, iLonInGrid))); // TODO 2024/6/19: 避免toArray操作减少内存复制
method.invoke(result, grid4D.toArray().getFloat(
index.set(0, levIndexGlobal-origin[1], latIndexGlobal-origin[2], lonIndexGlobal-origin[3])
));
} catch (ReflectiveOperationException e) { } catch (ReflectiveOperationException e) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e); throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e);
} catch (Exception e) { } catch (Exception e) {
@ -459,21 +410,6 @@ public class GfsDataServiceImpl implements IDataService {
return result; return result;
} }
private int getTargetTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
if (targetTime.isBefore(refTime)) {
throw new AppException(ErrorCode.QUERY_TIME_ERROR);
}
List<String> forecastHours = lastGridDataSetMeta.getForecastHours();
String targetHour = String.valueOf(getForecastHourFromTargetTime(targetTime, refTime));
if (!forecastHours.contains(targetHour)) {
throw new AppException(ErrorCode.QUERY_TIME_ERROR);
}
int iTime = forecastHours.indexOf(targetHour);
return iTime;
}
private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) { private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName()); long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
@ -492,7 +428,7 @@ public class GfsDataServiceImpl implements IDataService {
if (hour >= startHour) { if (hour >= startHour) {
iTimeList.add(i); iTimeList.add(i);
String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour)) String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour))
.format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); .format(Constant.API_TIME_FORMATTER);
timeList.add(timeStr); timeList.add(timeStr);
} }
if (iTimeList.size() >= 24) { if (iTimeList.size() >= 24) {
@ -508,64 +444,8 @@ public class GfsDataServiceImpl implements IDataService {
return map; return map;
} }
private int getLongitudeIndex(double lon) {
double[] lonList = tableConfig.getLonList();
if (lonList.length == 0) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "经度列表为空, 无法查找索引");
}
double minLon = lonList[0];
double maxLon = lonList[lonList.length - 1];
if (minLon > lon || lon > maxLon) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "经度不在有效范围内: " + minLon + " ~ " + maxLon);
}
int nearestIndex = NdArrayUtils.findNearestIndex(lonList, lon);
if (nearestIndex == -1) {
throw new AppException(ErrorCode.LONGITUDE_INDEX_ERROR, "找不到" + lon + "的经度索引");
}
return nearestIndex;
}
private int getLatitudeIndex(double lat) {
double[] latList = tableConfig.getLatList();
if (latList.length == 0) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "纬度列表为空, 无法查找索引");
}
double minLat = latList[0];
double maxLat = latList[latList.length - 1];
if (minLat > lat || lat > maxLat) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "纬度不在有效范围内: " + minLat + " ~ " + maxLat);
}
int nearestIndex = NdArrayUtils.findNearestIndex(latList, lat);
if (nearestIndex == -1) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + lat + "的纬度索引");
}
return nearestIndex;
}
private int getPressureIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfig.getPressureList()).boxed().collect(Collectors.toList());
if (CollectionUtils.isEmpty(levList)) {
throw new AppException(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空, 无法查找索引");
}
int index = levList.indexOf(pressureLevel);
if (index == -1) {
throw new AppException(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + pressureLevel + "的气压高度索引");
}
return index;
}
private int getLevelIndex(int level) {
if (level == SURFACE_LEVEL) {
return 0; // NOTE 2024/5/20: 0 表示地面风速对应10m高度其他变量对应2m高度
} else {
return getPressureIndex(level);
}
}
private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) { private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) {
return (int) Duration.between(refTime, targetTime).toHours(); return (int) Duration.between(refTime, targetTime).toHours();
} }
} }

@ -1,5 +1,6 @@
package com.htfp.weather.web.service.surfaceapi; package com.htfp.weather.web.service.surfaceapi;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils; import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.MeteoUtils; import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppException; import com.htfp.weather.web.exception.AppException;
@ -212,7 +213,7 @@ public class HeFengServiceImpl implements ISurfaceDataService {
* @return * @return
*/ */
private BaseHeFengData processHeFengData(BaseHeFengData data) { private BaseHeFengData processHeFengData(BaseHeFengData data) {
String format = "yyyy-MM-dd'T'HH:mmxxx"; String format = Constant.API_TIME_STRING;
if (data instanceof HeFengForecastHour) { if (data instanceof HeFengForecastHour) {
String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format); String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format);
((HeFengForecastHour) data).setFxTime(fxTime); ((HeFengForecastHour) data).setFxTime(fxTime);

@ -4,6 +4,7 @@ package com.htfp.weather.web.valid;
import javax.validation.Constraint; import javax.validation.Constraint;
import javax.validation.Payload; import javax.validation.Payload;
import java.lang.annotation.*; import java.lang.annotation.*;
import java.time.format.DateTimeFormatter;
/** /**
* @Author : shiyi * @Author : shiyi
@ -20,7 +21,7 @@ public @interface DateTimeStr {
String message() default "{javax.validation.constraints.DateTimeStr.message}"; String message() default "{javax.validation.constraints.DateTimeStr.message}";
String format() default "yyyy-MM-dd HH:mm:ss"; String format() default "";
Class<?>[] groups() default {}; Class<?>[] groups() default {};

@ -30,8 +30,12 @@ public class DateTimeValidator implements ConstraintValidator<DateTimeStr,String
return true; return true;
} }
String format = dateTimeStr.format(); String format = dateTimeStr.format();
DateTimeFormatter dateTimeFormatter;
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format); if (format == null || format.isEmpty()) {
dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; // 默认格式
} else {
dateTimeFormatter = DateTimeFormatter.ofPattern(format);
}
try { try {
dateTimeFormatter.parse(value); dateTimeFormatter.parse(value);

@ -1,5 +1,5 @@
server: server:
port: 10323 port: 10324
spring: spring:
profiles: profiles:
include: weather include: weather

@ -3,10 +3,10 @@
"metaTableName" : "gfs_meta_table", "metaTableName" : "gfs_meta_table",
"dataIndexName" : "gfs_data_index", "dataIndexName" : "gfs_data_index",
"metaIndexName" : "gfs_meta_table_index", "metaIndexName" : "gfs_meta_table_index",
"dataDir" : "./GFSData", "dataDir" : "./GFSData/",
"variableList" : [ "temp", "cloud", "wind360", "windSpeed"], "variableList" : [ "temp", "cloud", "windSpeed", "wind360", "humidity", "precip" ],
"lonSize" : 281, "lonSize" : 281,
"latSize" : 221, "latSize" : 221,
"levSize" : 10, "levSize" : 15,
"timeToLive" : 2 "timeToLive" : 1
} }
Loading…
Cancel
Save