package部分重新组织;冗余代码删除

refactor
shiyi 9 months ago
parent 8357fac67f
commit 2ec081e80f

@ -1,18 +0,0 @@
package com.htfp.weather.config;
import com.htfp.weather.griddata.common.TableConfig;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* @Author : shiyi
* @Date : 2024/1/22 19:02
* @Description :
*/
@Component("config")
public class Config {
@Resource
TableConfig tableConfig;
}

@ -8,8 +8,9 @@ import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.util.List;
/**
* @author shiyi
/** @Author : shiyi
* @Date : 2024/2/28 13:53
* @Description : GFS
*/
@Data
@ToString
@ -36,16 +37,10 @@ public abstract class BaseDataDownloader {
/**
*
*
* @param filesInfo
* @param fileInfoList
* @return
*/
public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
public BaseDataDownloader() {
}
}

@ -1,8 +1,6 @@
package com.htfp.weather.download;
package com.htfp.weather.download.gfs;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.JSONUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;

@ -1,10 +1,11 @@
package com.htfp.weather.download;
package com.htfp.weather.download.gfs;
import com.htfp.weather.download.BaseDataDownloader;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.HttpClientUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
@ -13,12 +14,7 @@ import ucar.nc2.NetcdfFiles;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.ConnectException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -30,6 +26,7 @@ import java.util.concurrent.Future;
/**
* @author shiyi
* @
*/
@Slf4j
@Component("gfsDownloader")
@ -42,7 +39,6 @@ public class GfsDownloader extends BaseDataDownloader {
private OffsetDateTime realUpdateDateTime;
public GfsDownloader(GfsDataConfig gfsDataConfig) {
super();
this.gfsDataConfig = gfsDataConfig;
}
@ -58,8 +54,8 @@ public class GfsDownloader extends BaseDataDownloader {
realUpdateDateTime = getAvailableUpdateDateTime(utcStartTime);
this.setRefTime(realUpdateDateTime);
this.setRefTimeStr(realUpdateDateTime.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)));
// 起报点和实际请求时间的差异
// 起报点和实际请求时间的差异
this.hourDiff = (int) Duration.between(realUpdateDateTime, utcStartTime).toHours();
log.info("[GFS Download] 起始为当前本地时间:{}", this.getStartTime());
log.info("[GFS Download] GFS最近更新时间UTC{},本地时间: {}", realUpdateDateTime, DateTimeUtils.getLocalZoneDateTime(realUpdateDateTime));
@ -69,24 +65,20 @@ public class GfsDownloader extends BaseDataDownloader {
public FileInfo download(FileInfo fileInfo) {
return download0(fileInfo, 2);
}
private FileInfo download0(FileInfo fileInfo, int retryNum) {
private FileInfo download0(FileInfo fileInfo, int retryNum) {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
File fileOut = new File(destDir, fileInfo.getFileName());
log.info("[GFS Download] 文件下载中,保存至 {}", fileOut);
try {
// TODO 2024/5/24: 增加重试机制
// if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
// throw new RuntimeException();
// }
// FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
// FIXME 2024/6/8: 如果连接是httpsjar包下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
// DONE: 改用okhttp FIXME 2024/6/8: 如果连接是httpsjar包启动下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
HttpClientUtils.downloadFileByUrl(url, fileOut.getPath());
log.info("[GFS Download] 文件下载成功: {}", fileOut);
fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
// DONE 2024/5/24: 文件服务器在外网不稳定,增加重试机制
fileInfo.setDownloadSuccess(false);
if (retryNum > 0) {
log.error("[GFS Download] 文件下载失败,重试中: {}", fileOut);
@ -99,6 +91,7 @@ public class GfsDownloader extends BaseDataDownloader {
return fileInfo;
}
/** 简单校验验证文件完整性,并生成索引文件*/
private boolean fileValid(String file) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
ncFile.getLocation();
@ -274,7 +267,7 @@ public class GfsDownloader extends BaseDataDownloader {
for (String variable : variables) {
stringJoiner.add(String.format("var_%s=on", variable));
}
stringJoiner.add("lev_surface=on"); // gust需要
stringJoiner.add("lev_surface=on"); // gust, prate需要
return stringJoiner.toString();
}

@ -1,4 +1,4 @@
package com.htfp.weather.info;
package com.htfp.weather.download.gfs;
/**
* @Author : shiyi
@ -8,6 +8,7 @@ package com.htfp.weather.info;
public enum GfsLevelsEnum {
//
SURFACE(9999, "地面"),
UPPER(0, "高空"),
PRES_1000hPa(1000, "1000 hPa"),
PRES_975hPa(975, "975 hPa"),
PRES_950hPa(950, "950 hPa"),

@ -1,4 +1,4 @@
package com.htfp.weather.griddata.common;
package com.htfp.weather.download.gfs;
import java.util.ArrayList;
import java.util.List;
@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
* @Description :
* @Description :
*/
public enum GfsVariableHeightEnum {
//

@ -1,4 +1,4 @@
package com.htfp.weather.griddata.common;
package com.htfp.weather.download.gfs;
import java.util.ArrayList;
import java.util.List;
@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
* @Description :
* @Description :
*/
public enum GfsVariableIsobaricEnum {
//

@ -2,7 +2,7 @@ package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;

@ -1,7 +1,7 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDataConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;

@ -1,9 +1,11 @@
package com.htfp.weather.griddata.utils;
package com.htfp.weather.griddata.common;
import lombok.Getter;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
/**
* @Author : shiyi

@ -7,8 +7,8 @@ import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Range;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.utils.ValueRange;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.griddata.common.ValueRange;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
@ -154,7 +154,19 @@ public class GfsDataFetcher extends BaseTableOperation {
Array array = gridDataSet.getVariable(variableName).toArray();
return array;
}
public GridDataSet getProfileByPressure(List<String> variableNameList, OffsetDateTime targetTime, double latitude, double longitude) throws Exception {
lastGridDataSetMeta = getLastGridDataSetMeta();
GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(lastGridDataSetMeta);
int iTime = getTargetTimeIndex(targetTime);
int iLat= getLatitudeIndex(latitude);
int iLon = getLongitudeIndex(longitude);
int[] origin = new int[]{iTime, 0, iLat, iLon};
int[] shape = new int[]{1,tableConfig.levSize, 1, 1};
// TODO 2024/6/17:
fetcher.setVariablesToGet(variableNameList);
fetcher.setOriginShape(origin, shape);
return fetcher.fetch();
}
/**
*
* @param dataSetId ID

@ -5,8 +5,9 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
@ -32,6 +33,8 @@ import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
* @author shi_y
@ -85,57 +88,64 @@ public class GfsDataImport extends BaseTableOperation {
* @throws Exception
*/
public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception {
meta.addAttribute("status", "DONE");
tableStoreGrid.updateDataSetMeta(meta);
return meta;
}
public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception {
meta.addAttribute("status", "DONE");
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
/** 导入指定起报时间的所有文件*/
public List<ImportResult> importData(OffsetDateTime refTime) throws Exception {
importing = true;
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES);
}
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
// datasetId和起报时间绑定
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
// done 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
// done 2024/5/13: 优化方案使用多线程并使用Future来获取结果但是必须保证每个时刻都成功导入
List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
// todo 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
List<ImportResult> finishedList;
try {
// List<Future<ImportResult>> futures = new ArrayList<>();
finishedList = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
// for(int i = 0; i < 200; i++) {
String file = fileList.get(i);
int iTime = i;
int forecastHour = getForecastHourFromFilename(file);
// futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
// 2024/5/13:使用多线程并使用Future来获取结果但是实际瓶颈可能是网络带宽而非计算
// futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime,forecastHour)));
ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour);
finishedList.add(importResult);
// ImportResult importResult = importFromNcFile(meta, file, 0, forecastHour);
if (importResult.isSuccess()) {
forecastHours.add(String.valueOf(importResult.getForecastHour()));
}
}
} finally {
importing = false;
}
// for (Future<ImportResult> future : futures) {
// ImportResult importResult = future.get();
// forecastHours.add(String.valueOf(importResult.getForcastHour()));
// finishedList.add(importResult);
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// if (importResult.isSuccess()) {
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// }
// }
List<ImportResult> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.warn("存在气象数据导入数据库失败,失败文件列表: {} ", failedList);
meta.addAttribute("status", "WRONG");
} else {
meta.addAttribute("status", "DONE");
}
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
@ -143,15 +153,16 @@ public class GfsDataImport extends BaseTableOperation {
putMeta(meta);
gfsDataFetcher.lastGridDataSetMeta = meta;
log.info("[tablestore]: 更新最新的数据元信息: {}", meta);
importing = false;
return finishedList;
}
/**
* read data from netcdf file and write data to table store.
*
* grib/nctablestore
* @param meta
* @param file netcdf
* @param iTime
* @param forecastHour
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
@ -191,7 +202,6 @@ public class GfsDataImport extends BaseTableOperation {
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, 0, grid2D);
}
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
@ -218,8 +228,9 @@ public class GfsDataImport extends BaseTableOperation {
}
}
/** 计算风速风向,包括地面坐标和气压坐标,并导入数据库*/
private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向需要保存到文件中
// TODO 2024/5/8: 风速风向最好单独保存为一个文件,如果后续需要二维可视化,可以直接本地读取出图,减少网络请求
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
Variable vwnd = ncFile.findVariable("v-component_of_wind" + suffix);

@ -1,9 +0,0 @@
package com.htfp.weather.griddata.operation;
/**
* @Author : shiyi
* @Date : 2024/2/2 13:47
* @Description :
*/
public interface IDataFetch {
}

@ -1,9 +1,8 @@
package com.htfp.weather.schedule;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
@ -23,7 +22,6 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@ -43,8 +41,6 @@ public class GridDataProcessor {
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataFetcher gfsDataFetcher;
@Resource
GfsDataConfig gfsDataConfig;
@Resource
ErrorReportService errorReportService;
@ -54,8 +50,10 @@ public class GridDataProcessor {
private final ExecutorService executorService = Executors.newFixedThreadPool(3);
boolean test = false;
/**每日北京时0,8,12,18点执行*/
@Scheduled(cron = "0 0 0,8,12,18 * * ?")
public void dailyDataProcess() throws Exception {
public void dailyDataProcess() {
if (download()) {
importToTableStore();
}

@ -9,7 +9,7 @@ import java.time.format.DateTimeFormatter;
* @Description :
*/
public class DateTimeUtils {
public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mmxxx";
public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx";
public static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ofPattern(DEFAULT_PATTERN);
/**
* ,

@ -89,7 +89,7 @@ public class HttpClientUtils {
}
public static boolean downloadFileByUrl(String sourceUrl, String localPath) throws IOException {
public static boolean downloadFileByUrl(String sourceUrl, String localPath) {
File file = new File(localPath);
Request request = new Request.Builder()
.url(sourceUrl)
@ -102,8 +102,9 @@ public class HttpClientUtils {
} else {
return false;
}
} catch (IOException e) {
return false;
}
return true;
}

@ -1,8 +1,8 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;

@ -29,7 +29,7 @@ public class SurfaceWeatherController {
@Resource(name = "hefeng")
ISurfaceDataService surfaceDataService;
@PostMapping("/now")
@PostMapping("/querySurfaceNowWeather")
public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
@ -41,8 +41,8 @@ public class SurfaceWeatherController {
return Result.success(nowWeatherStatus);
}
@PostMapping("/forecast")
public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception {
@PostMapping("/querySurfaceForecast")
public Result querySurfaceForecast(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
log.info("[data-server] 地面24小时预报结果查询 start: param={}", position2D);
@ -53,8 +53,8 @@ public class SurfaceWeatherController {
return Result.success(forecastSeries);
}
@PostMapping ("/warning")
public Result warning(@Validated @RequestBody Position2D position2D) throws Exception {
@PostMapping ("/queryWeatherWarning")
public Result queryWeatherWarning(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
log.info("[data-server] 地面气象预警信息查询 start: param={}", position2D);

@ -1,6 +1,5 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
@ -19,7 +18,7 @@ import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:15
* @Description :
* @Description :
*/
@CrossOrigin
@RestController @Slf4j
@ -61,8 +60,8 @@ public class UpperWeatherController {
return Result.success(nowWeatherByMultiPoint);
}
@PostMapping("/queryForecastTimeSeries24Hour")
public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) {
@PostMapping("/queryUpperForecast")
public Result queryUpperForecast(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
@ -72,8 +71,8 @@ public class UpperWeatherController {
return Result.success(forecastSeries);
}
@PostMapping("/queryForecastTimeSeriesInMultiPoints")
public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
@PostMapping("/queryUpperForecastInMultiPoints")
public Result queryUpperForecastInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
@ -93,6 +92,8 @@ public class UpperWeatherController {
return Result.success(nowWeatherByMultiPoint);
}
/**查询指定变量随气压的分布*/
@RequestMapping("/queryProfileByVariableAndPressure")
public Result queryProfileByVariableAndPressure(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
@ -106,22 +107,34 @@ public class UpperWeatherController {
return Result.success(profileResponse);
}
@PostMapping("/plane")
public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) {
/**查询全部变量随气压的分布*/
@RequestMapping("/queryProfileByPressure")
public Result queryProfileByAndPressure(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
log.info("[data-server] 高度廓线查询 start: param={}", profileRequest);
ProfileDataset profile = gfsDataService.getProfileByPressure(utcDateTime, latitude, longitude);
log.info("[data-server] 高度廓线查询 end");
return Result.success(profile);
}
@PostMapping("/queryPlaneGrid")
public Result queryPlaneGrid(@Validated @RequestBody PlaneRequest planeRequest) {
planeRequest.valid();
String variableName = planeRequest.getVariableName();
OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
int level = planeRequest.getLevel();
double minLat = planeRequest.getMinLat();
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
double minLat = planeRequest.getMinLatitude();
double maxLat = planeRequest.getMaxLatitude();
double minLon = planeRequest.getMinLongitude();
double maxLon = planeRequest.getMaxLongitude();
log.info("[data-server] 平面网格数据查询 start: param={}", planeRequest);
PlaneResponse forecastSeries = gfsDataService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
log.info("[data-server] 平面网格数据查询 end");
return Result.success(forecastSeries);
}
}

@ -1,6 +1,6 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDataConfig;
import lombok.Data;
/**

@ -1,8 +1,7 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.valid.DateTimeStr;
@ -35,29 +34,29 @@ public class PlaneRequest {
@NotNull
@Min(value = 70, message = "经度最小值为70东经180°")
@Max(value = 140, message = "经度最大值为140东经140°")
private Double minLon;
private Double minLongitude;
@NotNull
@Min(value = 70, message = "经度最小值为70东经180°")
@Max(value = 140, message = "经度最大值为140东经140°")
private Double maxLon;
private Double maxLongitude;
@NotNull
@Min(value = 0, message = "纬度最小值为0")
@Max(value = 55, message = "纬度最大值为55北纬55°")
private Double minLat;
private Double minLatitude;
@NotNull
@Min(value = 0, message = "纬度最小值为0")
@Max(value = 55, message = "纬度最大值为55北纬55°")
private Double maxLat;
private Double maxLatitude;
public void valid() {
// TODO 2024/4/21: 校验
if (minLon >= maxLon) {
if (minLongitude >= maxLongitude) {
throw new AppException(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度");
}
if (minLat >= maxLat) {
if (minLatitude >= maxLatitude) {
throw new AppException(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度");
}
}

@ -19,7 +19,6 @@ public class ProfileRequest {
@DateTimeStr(message = "时间格式错误")
String time;
@NotNull(message = "变量名不能为空")
String variableName;
@NotNull(message = "纬度 (latitude) 不能为空")

@ -0,0 +1,42 @@
package com.htfp.weather.web.pojo.response;
import lombok.Data;
import java.util.ArrayList;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:19
* @Description : 线
*/
@Data
public class ProfileDataset {
int[] pressureLevels;
// int[ pressureHeight;
double latitude;
double longitude;
float[] temp;
float[] windSpeed;
float[] wind360;
// List<String> windScale;
float[] humidity;
// float[] pressure;
float[] cloud;
float[] precip;
public ProfileDataset() {};
public ProfileDataset(int size) {
this.pressureLevels = new int[size];
// this.pressureHeight = pressureHeight;
temp = new float[size];
windSpeed = new float[size];
wind360 = new float[size];
// windScale = new float[size;
humidity = new float[size];
// pressure = new float[size;
cloud = new float[size];
precip = new float[size];
}
}

@ -4,21 +4,19 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.utils.ValueRange;
import com.htfp.weather.griddata.common.ValueRange;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.pojo.response.*;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import ucar.ma2.Array;
@ -30,7 +28,6 @@ import java.time.Duration;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.*;
/**
@ -201,9 +198,23 @@ public class GfsDataServiceImpl implements IDataService {
} catch (Exception e) {
throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage());
}
}
public ProfileDataset getProfileByPressure(OffsetDateTime targetTime, double latitude, double longitude) {
try {
List<String> variableNames = getVariableNamesInDatabase(GfsLevelsEnum.UPPER.getCode());
GridDataSet gridDataSet = gfsDataFetcher.getProfileByPressure(variableNames, targetTime, latitude, longitude);
ProfileDataset profile = buildProfileDataset(gridDataSet, GfsLevelsEnum.UPPER);
profile.setLatitude(latitude);
profile.setLongitude(longitude);
return profile;
} catch (AppException e) {
throw e;
} catch (Exception e) {
throw new AppException(ErrorCode.TABLESTORE_QUERY_ERROR, e.getMessage());
}
}
/**
*
*
@ -304,16 +315,7 @@ public class GfsDataServiceImpl implements IDataService {
// 利用反射构建数据集
Class<? extends TimeSeriesDataset> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
String variableNameInFile = entry.getKey();
String variableNameInApi;
if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
} else {
variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
}
if (variableNameInApi == null) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
}
String variableNameInApi = getVariableNameInApi(levelFlag, entry);
Grid4D grid4D = entry.getValue();
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
int[] shape = grid4D.getShape();
@ -381,16 +383,7 @@ public class GfsDataServiceImpl implements IDataService {
// 利用反射构建数据集
Class<? extends NowWeatherStatus> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
String variableNameInFile = entry.getKey();
String variableNameInApi;
if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
} else {
variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
}
if (variableNameInApi == null) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
}
String variableNameInApi = getVariableNameInApi(levelFlag, entry);
Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape());
int[] origin = grid4D.getOrigin(); // grid4D的起始坐标全局索引
try {
@ -410,6 +403,48 @@ public class GfsDataServiceImpl implements IDataService {
return result;
}
private ProfileDataset buildProfileDataset(GridDataSet gridDataSet, GfsLevelsEnum levelFlag) {
ProfileDataset result = new ProfileDataset();
// 利用反射构建数据集
Class<? extends ProfileDataset> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
try {
String variableNameInApi = getVariableNameInApi(levelFlag, entry);
Grid4D grid4D = entry.getValue();
int[] shape = grid4D.getShape();
int zSize = shape[1];
String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1);
Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType());
// TODO 2024/6/20: 这里的toArray操作在数据量较大的时候会比较吃内存和cpu
float[] values = (float[]) grid4D.toArray().section(
new int[]{0, 0, 0, 0}, new int[]{1, zSize, 1, 1}
).copyTo1DJavaArray();
method.invoke(result, values);
} catch (ReflectiveOperationException e) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE, e);
} catch (Exception e) {
e.printStackTrace();
throw new AppException(ErrorCode.RESPONSE_DATA_BUILD_ERROR);
}
}
result.setPressureLevels(tableConfig.getPressureList());
return result;
}
@NotNull
private static String getVariableNameInApi(GfsLevelsEnum levelFlag, Map.Entry<String, Grid4D> variable) {
String variableNameInFile = variable.getKey();
String variableNameInApi;
if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
} else {
variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
}
if (variableNameInApi == null) {
throw new AppException(ErrorCode.NO_SUCH_VARIABLE);
}
return variableNameInApi;
}
private Map<String, List> getFutureTimeIndexList(GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());

@ -15,6 +15,8 @@ import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -141,7 +143,9 @@ public class CaiYunServiceImpl implements ISurfaceDataService {
try {
TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(24);
for (int i = 0; i < 24; i++) {
timeSeriesDataset.getTime().add(hourly.getTemperature().get(i).getDatetime());
timeSeriesDataset.getTime().add(
OffsetDateTime.parse(hourly.getTemperature().get(i).getDatetime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)
);
timeSeriesDataset.getTemp()[i] = hourly.getTemperature().get(i).getValue();
timeSeriesDataset.getWindSpeed()[i] = (float) MeteoUtils.kmPerHour2mPerSecond(hourly.getWind().get(i).getSpeed());
timeSeriesDataset.getWind360()[i] = hourly.getWind().get(i).getDirection();

@ -20,6 +20,8 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import static com.htfp.weather.utils.HttpClientUtils.*;
@ -194,7 +196,9 @@ public class HeFengServiceImpl implements ISurfaceDataService {
List<SurfaceWeatherWarning> surfaceWeatherWarnings = new ArrayList<>();
for (HeFengWarning heFengWarning : heFengWarnings) {
SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning();
surfaceWeatherWarning.setPubTime(heFengWarning.getPubTime());
surfaceWeatherWarning.setPubTime(
OffsetDateTime.parse(heFengWarning.getPubTime()).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)
);
// surfaceWeatherWarning.setStartTime(heFengWarning.getStartTime());
surfaceWeatherWarning.setTitle(heFengWarning.getTitle());
surfaceWeatherWarning.setText(heFengWarning.getText());
@ -213,13 +217,12 @@ public class HeFengServiceImpl implements ISurfaceDataService {
* @return
*/
private BaseHeFengData processHeFengData(BaseHeFengData data) {
String format = Constant.API_TIME_STRING;
if (data instanceof HeFengForecastHour) {
String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format);
((HeFengForecastHour) data).setFxTime(fxTime);
String fxTime = ((HeFengForecastHour) data).getFxTime();
((HeFengForecastHour) data).setFxTime(OffsetDateTime.parse(fxTime).format(Constant.API_TIME_FORMATTER));
} else if (data instanceof HeFengNow) {
String obsTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengNow) data).getObsTime(), format);
((HeFengNow) data).setObsTime(obsTime);
String obsTime = ((HeFengNow) data).getObsTime();
((HeFengNow) data).setObsTime(OffsetDateTime.parse(obsTime).format(Constant.API_TIME_FORMATTER));
}
float windSpeed = (float) MeteoUtils.kmPerHour2mPerSecond(data.getWindSpeed());
data.setWindSpeed(windSpeed);

@ -1,11 +1,10 @@
package com.htfp.weather.download;
import com.htfp.weather.download.gfs.GfsDataConfig;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/4/21 16:28

@ -1,11 +1,11 @@
package com.htfp.weather.download;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDownloader;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/4/25 16:15

@ -1,6 +1,6 @@
package com.htfp.weather.griddata.operation;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@ -31,7 +31,6 @@ class GfsDataFetcherTest {
@Test
void getProfile() throws Exception {
log.info("获取廓线");
gfsDataFetcher.getProfile(dataSetId, GfsVariableIsobaricEnum.getGfsVariableName("Temp"), 9,0, 0);
}
// @Test

@ -1,30 +0,0 @@
package com.htfp.weather.griddata.utils;
import com.htfp.weather.utils.DateTimeUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/3/4 10:21
* @Description :
*/
class GfsUtilsTest {
@Test
void getRefTime() throws IOException {
String filename = "D:\\HTFP\\weather\\GFSData\\UTC-20240607.00\\BJT-20240607.17-from-UTC-20240607.00+9.grib2";
NetcdfFile ncFile = NetcdfFiles.open(filename);
String refTime = GfsUtils.getRefTime(ncFile);
String s = filename.split("\\+")[1];
Integer.parseInt(s.split(".grib2")[0]);
// assertEquals("2023-09-05T00:00+00:00", refTime);
// Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN));
}
}
Loading…
Cancel
Save