气象服务初版

refactor
shiyi 9 months ago
parent 7edc899527
commit f1ea254a27

@ -6,7 +6,7 @@
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.1-SNAPSHOT</version>
<repositories>
<repository>

@ -0,0 +1,12 @@
{
"duration" : 30,
"minLon" : 70.0,
"maxLon" : 140.0,
"minLat" : 0.0,
"maxLat" : 55.0,
"resolution" : 0.25,
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ],
"pressureLevels" : [ 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000 ],
"heightLevels" : [ 2, 10, 20, 30, 40, 50, 80, 100 ],
"saveRoot" : "./GFSData"
}

@ -47,15 +47,20 @@
</dependency>
<dependency>
<groupId>ai.djl</groupId>
<artifactId>api</artifactId>
<version>0.27.0</version>
</dependency>
<dependency>
<groupId>ai.djl.pytorch</groupId>
<artifactId>pytorch-engine</artifactId>
<version>0.27.0</version>
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.1-SNAPSHOT</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>ai.djl</groupId>-->
<!-- <artifactId>api</artifactId>-->
<!-- <version>0.27.0</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>ai.djl.pytorch</groupId>-->
<!-- <artifactId>pytorch-engine</artifactId>-->
<!-- <version>0.27.0</version>-->
<!-- </dependency>-->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@ -88,11 +93,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
@ -110,6 +110,10 @@
<artifactId>hibernate-validator</artifactId>
<version>6.0.16.Final</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- springmvc的参数valid校验依赖 结束 -->
</dependencies>
<dependencyManagement>
@ -141,8 +145,8 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<mainClass>com.htfp.weather.WeatherServiceApplication</mainClass>
<skip>true</skip>
<executable>true</executable>
<layout>JAR</layout>
</configuration>
<executions>
<execution>

@ -1,10 +1,8 @@
package com.htfp.weather.config;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigBean;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
/**
@ -15,6 +13,6 @@ import javax.annotation.Resource;
@Component("config")
public class Config {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
}

@ -25,6 +25,7 @@ public class FileInfo {
private String savePath;
private boolean isDownloadSuccess;
public FileInfo() {
}

@ -75,7 +75,8 @@ public class GfsDataConfig {
private String saveRoot;
@JsonIgnore
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json";
// private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json";
private final String configPath = System.getProperty("user.dir") + "/gfsDataConfig.json";
public void valid() {
// TODO 2024/4/21: 校验
if (minLon >= maxLon) {

@ -73,9 +73,6 @@ public class GfsDownloader extends BaseDataDownloader {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
if (!destDir.exists() && !destDir.mkdirs()) {
throw new RuntimeException("创建文件夹" + destDir + "失败");
}
File fileOut = new File(destDir, fileInfo.getFileName());
log.info("[GFS Download] 文件下载中,保存至 {}", fileOut);
@ -84,7 +81,9 @@ public class GfsDownloader extends BaseDataDownloader {
// if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
// throw new RuntimeException();
// }
FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
// FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
// FIXME 2024/6/8: 如果连接是httpsjar包下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
HttpClientUtils.downloadFileByUrl(url, fileOut.getPath());
log.info("[GFS Download] 文件下载成功: {}", fileOut);
fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
@ -134,6 +133,7 @@ public class GfsDownloader extends BaseDataDownloader {
String levelsStr = null;
String variablesStr = null;
try {
gfsDataConfig.valid();
lonlatBoxStr = fillUrlForLonLat();
levelsStr = fillUrlForLevels();
variablesStr = fillUrlForVariables();
@ -141,20 +141,26 @@ public class GfsDownloader extends BaseDataDownloader {
log.error("[GFS Download] 文件下载参数生成错误");
throw new RuntimeException(e);
}
String refTimeStr = this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING));
String savePath = gfsDataConfig.getSaveRoot() + "/" + refTimeStr;
File destDir = new File(savePath);
if (!destDir.exists() && !destDir.mkdirs()) {
throw new RuntimeException("创建文件夹" + destDir + "失败");
}
// 预报数据时间分辨率
final int forecastStep = 1;
// 分辨率为3h整数截断包括目标时刻本身
final int nFiles = 1 + gfsDataConfig.getDuration() / forecastStep;
// 存储文件信息
List<FileInfo> fileInfoList = new ArrayList<>(nFiles);
for (int i = 0; i < nFiles; i++) {
FileInfo fileInfo = new FileInfo();
int forecastHour = i * forecastStep + hourDiff;
fileInfo.setForecastHour(forecastHour);
String baseURL = getBaseURL(forecastHour, levelsStr, variablesStr, lonlatBoxStr);
fileInfo.setUrl(baseURL);
fileInfo.setRefTimeStr(this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)));
fileInfo.setRefTimeStr(refTimeStr);
fileInfo.setForecastUTCTimeStr(
this.getRefTime().plusHours(fileInfo.getForecastHour())
.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))
@ -164,7 +170,7 @@ public class GfsDownloader extends BaseDataDownloader {
.format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING))
);
fileInfo.setFileName(String.format("%s-from-%s+%d.grib2", fileInfo.getForecastBJTimeStr(), fileInfo.getRefTimeStr(), forecastHour));
fileInfo.setSavePath(gfsDataConfig.getSaveRoot() + "/" + fileInfo.getRefTimeStr());
fileInfo.setSavePath(savePath);
fileInfoList.add(fileInfo);
}
return fileInfoList;

@ -15,13 +15,15 @@ public enum GfsVariableHeightEnum {
RH("humidity", "Relative_humidity_cover_height_above_ground"),
WIND_SPEED("windSpeed", "Wind_speed_height_above_ground"),
WIND360("wind360", "Wind_direction_height_above_ground"),
GUST("gustSpeed", "Wind_speed_gust_surface"),
// GUST("gustSpeed", "Wind_speed_gust_surface"),
PRECIP("precip", "Precipitation_rate_surface"),
;
private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{
put("temp", "Temperature_height_above_ground");
put("humidity", "Relative_humidity_cover_height_above_ground");
put("windSpeed", "Wind_speed_height_above_ground");
put("wind360", "Wind_direction_height_above_ground");
put("precip", "Precipitation_rate_surface");
}};
String nameInApi;
String nameInFile;
@ -41,8 +43,17 @@ public enum GfsVariableHeightEnum {
public static String getGfsVariableName(GfsVariableHeightEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);
public static String getGfsVariableName(String nameInApi) {
return variableName.get(nameInApi);
}
public static String getVariableNameInApi(String nameInFile) {
for (GfsVariableHeightEnum anEnum : values()) {
if (anEnum.nameInFile.equals(nameInFile)) {
return anEnum.nameInApi;
}
}
return null;
}
public static List<String> getVariableNamesInApi() {

@ -30,6 +30,7 @@ public enum GfsVariableIsobaricEnum {
this.nameInFile = nameInFile;
}
public String getNameInApi() {
return nameInApi;
}
@ -44,6 +45,15 @@ public enum GfsVariableIsobaricEnum {
return variableName.get(tableVariableName);
}
public static String getVariableNameInApi(String nameInFile) {
for (GfsVariableIsobaricEnum anEnum : values()) {
if (anEnum.nameInFile.equals(nameInFile)) {
return anEnum.nameInApi;
}
}
return null;
}
public static List<String> getVariableNamesInApi() {
return new ArrayList<>(variableName.keySet());
}

@ -1,108 +1,143 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/15 19:10
* @Description : meta
*/
@Slf4j @Component
@Data @Component @Slf4j
@JsonIgnoreProperties(ignoreUnknown = true)
public class TableConfig {
@Autowired
@Resource
@JsonIgnore
GfsDataConfig dataConfig;
/**
* index
*/
public static String DATA_TABLE_NAME;
public static String META_TABLE_NAME;
public static String DATA_INDEX_NAME;
public static String META_INDEX_NAME;
@Value("duration")
public String dataTableName;
public String metaTableName;
public String dataIndexName;
public String metaIndexName;
/**
* ID
*/
public static String DATA_DIR;
public static List<String> VARIABLE_LIST;
public static int lonSize;
public static List<Double> lonList;
public static int latSize;
public static List<Double> latList;
public static int levSize;
public static List<Integer> levList;
public static int timeSizeMax = 72;
public static List<String> timeList;
public static DataType DATA_TYPE = DataType.FLOAT;
public String dataDir;
public List<String> variableList;
public int lonSize;
public int latSize;
public int levSize;
public int timeToLive;
// 数据过期时间,单位为秒
public static int TIME_TO_LIVE;
@JsonIgnore
public int timeSizeMax = 99;
@JsonIgnore
public double[] lonList;
@JsonIgnore
public double[] latList;
@JsonIgnore
public int[] pressureList;
@JsonIgnore
public int[] heightList;
// @JsonIgnore
// public List<Integer> forecastTimeList; // 预报时效列表
@JsonIgnore
public DataType dataType = DataType.FLOAT;
// note 修改作用范围在classes目录下resource文件下不改变
@JsonIgnore
// private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath();
private final String configPath = System.getProperty("user.dir") +"/tableConf.json";
@PostConstruct
private void initTableConfig() {
initDatasetConfig();
readConfig();
initLonList();
initLatList();
initLevList();
}
/**
*
*/
public static void initDatasetConfig() {
log.info("init table config...");
String path = Objects.requireNonNull(TableConfig.class.getClassLoader().getResource("config/tableConf.json")).getPath();
try (InputStream f = new FileInputStream(path)){
Gson gson = new Gson();
Map map = gson.fromJson(IOUtils.toString(f), Map.class);
DATA_TABLE_NAME = (String) map.get("dataTableName");
META_TABLE_NAME = (String) map.get("metaTableName");
DATA_INDEX_NAME = (String) map.get("dataIndexName");
META_INDEX_NAME = (String) map.get("metaIndexName");
DATA_DIR = (String) map.get("dataDir");
VARIABLE_LIST = (ArrayList<String>) map.get("variableList");
lonSize = ((Double) map.get("lonSize")).intValue();
latSize = ((Double) map.get("latSize")).intValue();
levSize = ((Double) map.get("levSize")).intValue();
TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天
public void readConfig() {
// String pathSeparator = System.getProperty("file.separator");
try (InputStream f = new FileInputStream(configPath)){
String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8);
TableConfig tableConfig = JSONUtils.json2pojo(jsonStr, TableConfig.class);
this.dataTableName = tableConfig.getDataTableName();
this.metaTableName = tableConfig.getMetaTableName();
this.dataIndexName = tableConfig.getDataIndexName();
this.metaIndexName = tableConfig.getMetaIndexName();
this.variableList = tableConfig.getVariableList();
this.dataDir = tableConfig.getDataDir();
this.lonSize = tableConfig.getLonSize();
this.latSize = tableConfig.getLatSize();
this.levSize = dataConfig.getPressureLevels().length;
this.timeToLive = tableConfig.getTimeToLive();
log.info("[config] 读取 DataTable 配置 : {}", configPath);
} catch (NoSuchFileException e) {
log.error("[config] 配置文件{}不存在", configPath, e);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public void writeConfig(TableConfig updateConfig) throws IOException {
writeConfig(updateConfig, configPath);
}
public void writeConfig(TableConfig updateConfig, String filePath) throws IOException {
this.dataTableName = updateConfig.getDataTableName();
this.metaTableName = updateConfig.getMetaTableName();
this.dataIndexName = updateConfig.getDataIndexName();
this.metaIndexName = updateConfig.getMetaIndexName();
this.variableList = updateConfig.getVariableList();
this.dataDir = updateConfig.getDataDir();
this.lonSize = updateConfig.getLonSize();
this.latSize = updateConfig.getLatSize();
this.levSize = updateConfig.getLevSize();
// // FIXME 2024/5/10: 维度信息取决于数据下载,目前需要两个配置一起更新
this.timeToLive = updateConfig.getTimeToLive();
JSONUtils.pojo2jsonFile(this, filePath);
log.info("配置文件 {} 更新为: {}", filePath, updateConfig);
}
public void valid() {
}
private void initLonList() {
double lonStart = dataConfig.getMinLon();
double res = dataConfig.getResolution();
lonList = new ArrayList<>();
lonList = new double[lonSize];
for (int i = 0; i < lonSize; i++) {
lonList.add(lonStart + i * res);
lonList[i] = lonStart + i * res;
}
}
private void initLatList() {
double latStart = dataConfig.getMinLat();
double res = dataConfig.getResolution();
latList = new ArrayList<>();
latList = new double[latSize];
for (int i = 0; i < latSize; i++) {
latList.add(latStart + i * res);
latList[i] = latStart + i * res;
}
}
private void initLevList() {
levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList());
}
pressureList = dataConfig.getPressureLevels();
heightList = dataConfig.getHeightLevels();
levSize = pressureList.length + heightList.length;
}
}

@ -1,143 +0,0 @@
package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.util.List;
import java.util.Objects;
@Data @Component @Slf4j
@JsonIgnoreProperties(ignoreUnknown = true)
public class TableConfigBean {
@Resource
@JsonIgnore
GfsDataConfig dataConfig;
@Value("duration")
public String dataTableName;
public String metaTableName;
public String dataIndexName;
public String metaIndexName;
/**
* ID
*/
public String dataDir;
public List<String> variableList;
public int lonSize;
public int latSize;
public int levSize;
public int timeToLive;
@JsonIgnore
public int timeSizeMax = 99;
@JsonIgnore
public double[] lonList;
@JsonIgnore
public double[] latList;
@JsonIgnore
public int[] pressureList;
@JsonIgnore
public int[] heightList;
// @JsonIgnore
// public List<Integer> forecastTimeList; // 预报时效列表
@JsonIgnore
public DataType dataType = DataType.FLOAT;
// note 修改作用范围在classes目录下resource文件下不改变
@JsonIgnore
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath();
@PostConstruct
private void initTableConfig() {
readConfig();
initLonList();
initLatList();
initLevList();
}
public void readConfig() {
// String pathSeparator = System.getProperty("file.separator");
try (InputStream f = new FileInputStream(configPath)){
String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8);
TableConfigBean tableConfig = JSONUtils.json2pojo(jsonStr, TableConfigBean.class);
this.dataTableName = tableConfig.getDataTableName();
this.metaTableName = tableConfig.getMetaTableName();
this.dataIndexName = tableConfig.getDataIndexName();
this.metaIndexName = tableConfig.getMetaIndexName();
this.variableList = tableConfig.getVariableList();
this.dataDir = tableConfig.getDataDir();
this.lonSize = tableConfig.getLonSize();
this.latSize = tableConfig.getLatSize();
this.levSize = dataConfig.getPressureLevels().length;
this.timeToLive = tableConfig.getTimeToLive();
log.info("[config] 读取 DataTable 配置 : {}", configPath);
} catch (NoSuchFileException e) {
log.error("[config] 配置文件{}不存在", configPath, e);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public void writeConfig(TableConfigBean updateConfig) throws IOException {
writeConfig(updateConfig, configPath);
}
public void writeConfig(TableConfigBean updateConfig, String filePath) throws IOException {
this.dataTableName = updateConfig.getDataTableName();
this.metaTableName = updateConfig.getMetaTableName();
this.dataIndexName = updateConfig.getDataIndexName();
this.metaIndexName = updateConfig.getMetaIndexName();
this.variableList = updateConfig.getVariableList();
this.dataDir = updateConfig.getDataDir();
this.lonSize = updateConfig.getLonSize();
this.latSize = updateConfig.getLatSize();
this.levSize = updateConfig.getLevSize();
// // FIXME 2024/5/10: 维度信息取决于数据下载,目前需要两个配置一起更新
this.timeToLive = updateConfig.getTimeToLive();
JSONUtils.pojo2jsonFile(this, filePath);
log.info("配置文件 {} 更新为: {}", filePath, updateConfig);
}
public void valid() {
}
private void initLonList() {
double lonStart = dataConfig.getMinLon();
double res = dataConfig.getResolution();
lonList = new double[lonSize];
for (int i = 0; i < lonSize; i++) {
lonList[i] = lonStart + i * res;
}
}
private void initLatList() {
double latStart = dataConfig.getMinLat();
double res = dataConfig.getResolution();
latList = new double[latSize];
for (int i = 0; i < latSize; i++) {
latList[i] = latStart + i * res;
}
}
private void initLevList() {
pressureList = dataConfig.getPressureLevels();
heightList = dataConfig.getHeightLevels();
levSize = pressureList.length + heightList.length;
}
}

@ -0,0 +1,106 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
import javax.annotation.PostConstruct;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Author : shiyi
* @Date : 2024/1/15 19:10
* @Description : meta
*/
@Slf4j @Component
public class TableConfigStatic {
@Autowired
GfsDataConfig dataConfig;
/**
* index
*/
public static String DATA_TABLE_NAME;
public static String META_TABLE_NAME;
public static String DATA_INDEX_NAME;
public static String META_INDEX_NAME;
/**
* ID
*/
public static String DATA_DIR;
public static List<String> VARIABLE_LIST;
public static int lonSize;
public static List<Double> lonList;
public static int latSize;
public static List<Double> latList;
public static int levSize;
public static List<Integer> levList;
public static int timeSizeMax = 72;
public static List<String> timeList;
public static DataType DATA_TYPE = DataType.FLOAT;
// 数据过期时间,单位为秒
public static int TIME_TO_LIVE;
@PostConstruct
private void initTableConfig() {
initDatasetConfig();
initLonList();
initLatList();
initLevList();
}
/**
*
*/
public static void initDatasetConfig() {
log.info("init table config...");
String path = System.getProperty("user.dir") + "./tableConf.json";
try (InputStream f = new FileInputStream(path)){
Gson gson = new Gson();
Map map = gson.fromJson(IOUtils.toString(f), Map.class);
DATA_TABLE_NAME = (String) map.get("dataTableName");
META_TABLE_NAME = (String) map.get("metaTableName");
DATA_INDEX_NAME = (String) map.get("dataIndexName");
META_INDEX_NAME = (String) map.get("metaIndexName");
DATA_DIR = (String) map.get("dataDir");
VARIABLE_LIST = (ArrayList<String>) map.get("variableList");
lonSize = ((Double) map.get("lonSize")).intValue();
latSize = ((Double) map.get("latSize")).intValue();
levSize = ((Double) map.get("levSize")).intValue();
TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
private void initLonList() {
double lonStart = dataConfig.getMinLon();
double res = dataConfig.getResolution();
lonList = new ArrayList<>();
for (int i = 0; i < lonSize; i++) {
lonList.add(lonStart + i * res);
}
}
private void initLatList() {
double latStart = dataConfig.getMinLat();
double res = dataConfig.getResolution();
latList = new ArrayList<>();
for (int i = 0; i < latSize; i++) {
latList.add(latStart + i * res);
}
}
private void initLevList() {
levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList());
}
}

@ -3,14 +3,10 @@ package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.TableStoreGridConfig;
import com.aliyun.tablestore.grid.TableStoreGrid;
import com.google.common.collect.Tables;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableStoreConf;
import org.checkerframework.checker.units.qual.A;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@ -20,7 +16,7 @@ public abstract class BaseTableOperation{
protected TableStoreGrid tableStoreGrid;
// private String pathSeperator = "/";
@Autowired
private TableConfigBean tableConfigBean;
private TableConfig tableConfig;
@Autowired
private TableStoreConf tableStoreConf;
@ -31,8 +27,8 @@ public abstract class BaseTableOperation{
config.setAccessId(tableStoreConf.getAccessId());
config.setAccessKey(tableStoreConf.getAccessKey());
config.setTableStoreInstance(tableStoreConf.getInstanceName());
config.setDataTableName(tableConfigBean.getDataTableName());
config.setMetaTableName(tableConfigBean.getMetaTableName());
config.setDataTableName(tableConfig.getDataTableName());
config.setMetaTableName(tableConfig.getMetaTableName());
tableStoreGrid = new TableStoreGrid(config);
}

@ -4,7 +4,7 @@ import com.alicloud.openservices.tablestore.model.TableOptions;
import com.alicloud.openservices.tablestore.model.search.FieldSchema;
import com.alicloud.openservices.tablestore.model.search.FieldType;
import com.alicloud.openservices.tablestore.model.search.IndexSchema;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigStatic;
import java.util.Arrays;
@ -21,7 +21,7 @@ public class CreateTable extends BaseTableOperation {
* @throws Exception
*/
private void createStore() throws Exception {
TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE, 1);
TableOptions tableOptions = new TableOptions(TableConfigStatic.TIME_TO_LIVE, 1);
this.tableStoreGrid.createStore(tableOptions);
}
@ -38,7 +38,7 @@ public class CreateTable extends BaseTableOperation {
new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("ref_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true)
));
this.tableStoreGrid.createMetaIndex(TableConfig.META_INDEX_NAME, indexSchema);
this.tableStoreGrid.createMetaIndex(TableConfigStatic.META_INDEX_NAME, indexSchema);
}
public static void main(String[] args) throws Exception {

@ -6,7 +6,7 @@ import com.aliyun.tablestore.grid.GridDataDeleter;
import com.aliyun.tablestore.grid.core.RequestBuilder;
import com.aliyun.tablestore.grid.model.DeleteDataParam;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigStatic;
import lombok.extern.slf4j.Slf4j;
@ -58,7 +58,7 @@ public class DataDeleter extends BaseTableOperation {
Set<Integer> refTimes = dataSetMeta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet());
for (int t=0;t<3;t++) {
for (int z=0; z<8; z++) {
DeleteDataParam param = new DeleteDataParam(TableConfig.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z);
DeleteDataParam param = new DeleteDataParam(TableConfigStatic.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z);
//添加到batch操作中。
batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange());
}
@ -86,7 +86,7 @@ public class DataDeleter extends BaseTableOperation {
AsyncClient client = getClient();
//设置数据表名称。
RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfig.DATA_TABLE_NAME);
RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfigStatic.DATA_TABLE_NAME);
//设置起始主键。
PrimaryKeyBuilder primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder();

@ -3,14 +3,13 @@ package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import ucar.ma2.Array;
import javax.annotation.Resource;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -20,10 +19,10 @@ import java.util.List;
* @Description :
*/
@Slf4j @Component
@DependsOn({"tableStoreConf","tableConfigBean"})
@DependsOn({"tableStoreConf", "tableConfig"})
public class GfsDataFetcher extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
/**
* (time, lev, lat, lon)
@ -60,7 +59,7 @@ public class GfsDataFetcher extends BaseTableOperation {
*/
public Array getPlane(String dataSetId, String variable, int iTime, int iLev) throws Exception {
int[] origin = new int[] {iTime, iLev, 0, 0};
int[] shape = new int[] {1, 1, tableConfigBean.latSize, tableConfigBean.lonSize};
int[] shape = new int[] {1, 1, tableConfig.latSize, tableConfig.lonSize};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable),
origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();
@ -100,7 +99,7 @@ public class GfsDataFetcher extends BaseTableOperation {
*/
public Array getProfile(String dataSetId, String variable, int iTime, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTime, 0, iLat, iLon};
int[] shape = new int[] {1, tableConfigBean.levSize, 1, 1};
int[] shape = new int[] {1, tableConfig.levSize, 1, 1};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable),
origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();

@ -7,13 +7,15 @@ import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@ -30,17 +32,19 @@ import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* @author shi_y
*/
@Slf4j
@Component
@DependsOn("tableConfigBean")
@DependsOn("tableConfig")
public class GfsDataImport extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
@Getter
boolean importing = false;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
/**
@ -92,7 +96,7 @@ public class GfsDataImport extends BaseTableOperation {
}
public void importData(OffsetDateTime refTime) throws Exception {
public List<ImportResult> importData(OffsetDateTime refTime) throws Exception {
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
@ -100,32 +104,43 @@ public class GfsDataImport extends BaseTableOperation {
throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES);
}
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfigBean.variableList);
int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape);
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
// done 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
// done 2024/5/13: 优化方案使用多线程并使用Future来获取结果但是必须保证每个时刻都成功导入
List<Future<ImportResult>> futures = new ArrayList<>();
List<ImportResult> finishedList;
try {
finishedList = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
// for(int i = 0; i < 200; i++) {
String file = fileList.get(i);
int iTime = i;
futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
// ImportResult importResult = importFromNcFile(meta, file, iTime);
// forecastHours.add(String.valueOf(importResult.getForcastHour()));
int forecastHour = getForecastHourFromFilename(file);
// futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour);
finishedList.add(importResult);
// ImportResult importResult = importFromNcFile(meta, file, 0, forecastHour);
if (importResult.isSuccess()) {
forecastHours.add(String.valueOf(importResult.getForecastHour()));
}
}
for (Future<ImportResult> future : futures) {
ImportResult importResult = future.get();
forecastHours.add(String.valueOf(importResult.getForcastHour()));
} finally {
importing = false;
}
// for (Future<ImportResult> future : futures) {
// ImportResult importResult = future.get();
// forecastHours.add(String.valueOf(importResult.getForcastHour()));
// }
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
return finishedList;
}
/**
@ -135,13 +150,17 @@ public class GfsDataImport extends BaseTableOperation {
* @param file netcdf
* @param iTime
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) {
int time = 0;
try {
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
NetcdfFile ncFile = NetcdfFiles.open(file);
// 相对reftime的小时数文件可能缺失因此time可能是不连续的但是iTime是连续的
time = (int) ncFile.findVariable("time").read().getDouble(0);
if (forecastHour == -1) {
forecastHour = (int) ncFile.findVariable("time").read().getDouble(0);
}
int xsize = meta.getxSize();
int ysize = meta.getySize();
for (String variableName : meta.getVariables()) {
if ("Wind_speed_gust_surface".equals(variableName) || "Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)
|| "Wind_speed_height_above_ground".equals(variableName) || "Wind_direction_height_above_ground".equals(variableName)) {
@ -155,26 +174,33 @@ public class GfsDataImport extends BaseTableOperation {
int zSize = variable.getShape(1);
for (int z = 0; z < zSize; z++) {
// 注意高度索引是递增排序
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, z, grid2D);
}
} // TODO 2024/5/23: 导入不含高度坐标的数据
} else if (shapeLength == 3) {
// DONE 2024/5/24: 导入不含高度坐标的数据
Array array = variable.read(new int[]{0, 0, 0}, new int[]{1,xsize, ysize});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, 0, grid2D);
}
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
}
// 导入风速风向
importWind(meta, ncFile, iTime);
ncFile.close();
importWind(writer, meta, ncFile, iTime);
} catch (Exception e) {
log.error("[tablestore] 导入文件数据失败,_t={}: {}", iTime, file, e);
return new ImportResult(false, file, time, iTime);
log.error("[tablestore] 导入文件数据失败,_t={}: {}", forecastHour, file, e);
return new ImportResult(false, file, forecastHour, iTime);
}
log.info("[tablestore] 导入文件数据成功,_t={}: {}", iTime, file);
return new ImportResult(true, file, time, iTime);
log.info("[tablestore] 导入文件数据成功,_t={}: {}", forecastHour, file);
return new ImportResult(true, file, forecastHour, iTime);
}
private void transferUnit(String variableName, Array array) {
@ -184,9 +210,12 @@ public class GfsDataImport extends BaseTableOperation {
if (GfsVariableHeightEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
if (GfsVariableHeightEnum.PRECIP.getNameInFile().equals(variableName)) {
MeteoUtils.precipRate2mmPerHour(array);
}
}
private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向需要保存到文件中
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
@ -197,7 +226,6 @@ public class GfsDataImport extends BaseTableOperation {
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = uwnd.getShape(1);
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
@ -214,7 +242,19 @@ public class GfsDataImport extends BaseTableOperation {
}
}
}
/** 从文件名中获取预报时效 */
private int getForecastHourFromFilename(String filename) {
try {
String s = filename.split("\\+")[1].split(".grib2")[0];
if (StringUtils.isEmpty(s)) {
return -1;
} else {
return Integer.parseInt(s);
}
} catch (Exception e) {
return -1;
}
}
private List<String> getFileVariables(List<String> variables) {
List<String> fileVariables = new ArrayList<>();
for (String variable : variables) {
@ -243,7 +283,7 @@ public class GfsDataImport extends BaseTableOperation {
*/
private List<String> getFiles(OffsetDateTime refTime) {
String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File fileDir = new File(tableConfigBean.dataDir + dataFolder);
File fileDir = new File(tableConfig.dataDir, dataFolder);
if (!fileDir.exists()) {
log.warn("文件夹 {} 不存在", fileDir);
return null;
@ -271,13 +311,13 @@ public class GfsDataImport extends BaseTableOperation {
public static class ImportResult {
private boolean success;
private String file;
private int forcastHour;
private int forecastHour;
private int iTime;
public ImportResult(boolean success, String file, int forcastHour, int iTime) {
public ImportResult(boolean success, String file, int forecastHour, int iTime) {
this.success = success;
this.file = file;
this.forcastHour = forcastHour;
this.forecastHour = forecastHour;
this.iTime = iTime;
}
}

@ -7,7 +7,7 @@ import com.aliyun.tablestore.grid.core.QueryBuilder;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@ -23,7 +23,7 @@ import java.util.List;
@Component
public class QueryMeta extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
/**
*
*
@ -33,7 +33,7 @@ public class QueryMeta extends BaseTableOperation {
public GridDataSetMeta getLastGridDataSetMeta() throws Exception {
QueryGridDataSetResult result = tableStoreGrid.queryDataSets(
tableConfigBean.metaIndexName,
tableConfig.metaIndexName,
QueryBuilder.and()
.equal("status", "DONE")
.build(),

@ -1,7 +1,7 @@
package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.model.TableOptions;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigStatic;
/**
* @Author : shiyi
@ -11,12 +11,12 @@ import com.htfp.weather.griddata.common.TableConfig;
public class UpdateTable extends BaseTableOperation {
private void update() throws Exception {
TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE);
TableOptions tableOptions = new TableOptions(TableConfigStatic.TIME_TO_LIVE);
this.tableStoreGrid.updateStoreOption(tableOptions);
}
public static void main(String[] args) {
TableConfig.initDatasetConfig();
TableConfigStatic.initDatasetConfig();
UpdateTable example = new UpdateTable();
try {
example.update();

@ -43,6 +43,15 @@ public enum GfsLevelsEnum {
public String getInfo() {
return info;
}
public static GfsLevelsEnum getByCode(Integer code) {
for (GfsLevelsEnum c : GfsLevelsEnum.values()) {
if (c.getCode().equals(code)) {
return c;
}
}
return null;
}
public static boolean contains(Integer test) {
for (GfsLevelsEnum c : GfsLevelsEnum.values()) {
if (c.getCode().equals(test)) {

@ -6,16 +6,15 @@ import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.info.Constant;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.service.FileService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.checkerframework.checker.units.qual.A;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
@ -30,7 +29,7 @@ import java.util.List;
@Component
@Slf4j
@DependsOn({"gfsDataImport", "gfsDownloader"})
public class GridDataProcesser {
public class GridDataProcessor {
@Resource
GfsDataImport gfsDataImport;
@Resource
@ -38,16 +37,17 @@ public class GridDataProcesser {
@Resource
GfsDataConfig gfsDataConfig;
public void dailyDataProcess() throws Exception {
gfsDownloader.iniTimeSetting();
OffsetDateTime refTime = gfsDownloader.getRefTime();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
// if (allComplete) {
try {
gfsDataImport.importData(refTime);
gfsDataImport.importData(refTime)
} catch (AppExcpetion e) {
log.info("导入数据失败");
// }
}
}

@ -49,8 +49,8 @@ public class DateTimeUtils {
}
public static OffsetDateTime getUTCDateTime(OffsetDateTime localDateTime) {
return localDateTime.withOffsetSameInstant(ZoneOffset.UTC);
public static OffsetDateTime getUTCDateTime(OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
// public static LocalDateTime getUTCDateTime(String timeString, String formatStr) {

@ -9,9 +9,13 @@ import okhttp3.ResponseBody;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -27,7 +31,8 @@ public class HttpClientUtils {
.readTimeout(60, TimeUnit.SECONDS)
.writeTimeout(60, TimeUnit.SECONDS)
.build();
private static final OkHttpClient DONLOAD_OKHTTP_CLIENT = new OkHttpClient.Builder()
.build();
// public static <T> T sendGet(String url, Map<String, String> params, Class<T> responseClass) {
// MultiValueMap<String, String> queryParams = new LinkedMultiValueMap<>();
// if (params != null) {
@ -83,4 +88,23 @@ public class HttpClientUtils {
}
}
public static boolean downloadFileByUrl(String sourceUrl, String localPath) throws IOException {
File file = new File(localPath);
Request request = new Request.Builder()
.url(sourceUrl)
.build();
try (Response response = DONLOAD_OKHTTP_CLIENT.newCall(request).execute()) {
if (response.code() == 200) {
InputStream stream = response.body().byteStream();
Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
} else {
return false;
}
}
return true;
}
}

@ -1,17 +1,8 @@
package com.htfp.weather.utils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import org.checkerframework.checker.units.qual.A;
import ucar.ma2.Array;
import ucar.ma2.ArrayFloat;
import ucar.ma2.Index;
import ucar.nc2.units.DateType;
import java.nio.ByteBuffer;
import java.text.DecimalFormat;
import java.util.Arrays;
/**
@ -46,7 +37,16 @@ public class MeteoUtils {
// public static double height2Pressure(double pressure) {
//
// }
/**
* @param array (kg/m^2/s)
* @return mm/h
*/
public static Array precipRate2mmPerHour(Array array) {
for (int i = 0; i < array.getSize(); i++) {
array.setFloat(i, array.getFloat(i)*3600);
}
return array;
}
/**
*
*/

@ -0,0 +1,9 @@
package com.htfp.weather.web.config;
/**
* @Author : shiyi
* @Date : 2024/6/12 14:12
* @Description :
*/
public class LogAspect {
}

@ -3,19 +3,27 @@ package com.htfp.weather.web.controller;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.GfsConfigUpdate;
import com.htfp.weather.web.pojo.request.TableConfigUpdate;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.service.FileService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -31,48 +39,66 @@ import java.util.stream.Collectors;
@RequestMapping("/htfp/weather/config")
public class ConfigController {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
@Resource
GfsDataConfig gfsDataConfig;
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataImport gfsDataImport;
@Resource
FileService fileService;
private final String SECRET = "htfpweather";
private void validSecret(String secret) {
if (!SECRET.equals(secret)) {
if (StringUtils.isEmpty(secret) || !SECRET.equals(secret)) {
throw new AppExcpetion(ErrorCode.SECRET_ERROR);
}
}
/**
* tablestore
*/
@RequestMapping("/queryDatabaseConfig")
public Result queryDatabaseConfig() {
return Result.success(tableConfigBean);
return Result.success(tableConfig);
}
/**
* tablestore
*/
@RequestMapping("/updateDatabaseConfig")
public Result updateDatabaseConfig(@RequestParam String secret, @Validated @RequestBody TableConfigBean updateConfig) {
public Result updateDatabaseConfig(@RequestBody TableConfigUpdate request) {
String secret = request.getSecret();
TableConfig updateConfig = request.getConfig();
validSecret(secret);
if (updateConfig == null) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空");
}
updateConfig.valid();
try {
tableConfigBean.writeConfig(updateConfig);
tableConfig.writeConfig(updateConfig);
} catch (IOException e) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e);
}
return Result.success();
}
/**
* GFS
*/
@RequestMapping("/queryDataSourceConfig")
public Result queryDataSourceConfig() {
return Result.success(gfsDataConfig);
}
/**
* GFS
*/
@RequestMapping("/updateDataSourceConfig")
public Result updateDataSourceConfig(@RequestParam String secret, @Validated @RequestBody GfsDataConfig updateConfig) {
public Result updateDataSourceConfig(@Validated @RequestBody GfsConfigUpdate request) {
String secret = request.getSecret();
GfsDataConfig updateConfig = request.getConfig();
validSecret(secret);
if (updateConfig == null) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空");
@ -86,8 +112,12 @@ public class ConfigController {
return Result.success();
}
@RequestMapping("/download")
public Result download(@RequestParam String secret) {
/**
* 24
*/
@RequestMapping("/downloadAllFiles")
public Result downloadAllFiles(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
// List<FileInfo> allCompleted = false;
try {
@ -104,24 +134,120 @@ public class ConfigController {
} else {
return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data);
}
} catch (Exception e) {
} catch (AppExcpetion e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
}
/**
*
*/
@RequestMapping("/downloadSingleFile")
public Result downloadSingleFile(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime offsetDateTime = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING));
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo(); // TODO 2024/6/12: 可以优化不用生成24小时所有的文件信息
for (FileInfo fileInfo : fileInfoList) {
if (fileInfo.getForecastUTCTimeStr().equals(targetUtcStr)) {
FileInfo downloadResult = gfsDownloader.download(fileInfo);
if (downloadResult.isDownloadSuccess()) {
return Result.success(downloadResult);
} else {
return Result.error(ErrorCode.DOWNLOAD_ERROR);
}
}
}
return Result.error(ErrorCode.QUERY_TIME_ERROR, ": 请选择当前时间之后24小时范围内的时刻");
} catch (AppExcpetion e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
}
@RequestMapping("/import")
/**
*
*/
@RequestMapping("/importToTablestoreByReferenceTime")
public Result importData(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"));
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
try {
gfsDataImport.importData(time);
return Result.success();
List<ImportResult> finishedList = gfsDataImport.importData(time);
List<ImportResult> successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<ImportResult> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
Map<String, List<ImportResult>> data = new HashMap<>();
data.put("successList", successList);
data.put("failedList", failedList);
if (CollectionUtils.isEmpty(failedList)) {
return Result.success(data);
} else {
return new Result(false, ErrorCode.TABLESTORE_IMPORT_ERROR.getCode(), ErrorCode.TABLESTORE_IMPORT_ERROR.getMsg() + ": 数据导入未全部完成", data);
}
} catch (AppExcpetion e) {
return Result.error(e.getErrorCode());
} catch (Exception e) {
e.printStackTrace();
return Result.error(e);
}
}
/**
*
*/
@RequestMapping("/queryAllFileByReferenceTime")
public Result queryAllFileByReferenceTime(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr);
// 查找目录下的文件
List<com.htfp.weather.web.pojo.response.FileInfo> fileList = fileService.getFileList(dataDir);
return Result.success(fileList);
}
/**
*
*/
@RequestMapping("/deleteDirByReferenceTime")
public Result deleteDirByReferenceTime(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
String timeStr = DateTimeUtils.getUTCDateTime(time).format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr);
if (gfsDataImport.isImporting()) {
return Result.error(ErrorCode.CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD);
}
try {
fileService.deleteDir(dataDir);
return Result.success();
} catch (IOException e) {
return Result.error(e);
}
}
/**
*
*/
@RequestMapping("/deleteTargetFile")
public Result deleteTargetFile(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
String filePath = params.get("filePath");
File file = new File(filePath);
if (gfsDataImport.isImporting()) {
return Result.error(ErrorCode.CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD);
}
try {
fileService.deleteFile(file);
return Result.success();
} catch (IOException e) {
return Result.error(e);
}
}
}

@ -1,10 +1,9 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
@ -16,14 +15,14 @@ import javax.annotation.Resource;
@Controller
public class ThymeleafTest {
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
@GetMapping("index")// 页面的url地址
public String getindex(Model model, String secret) {
if (!"shiyi".equals(secret)){
return "wrong";
}
model.addAttribute("tableConfig", tableConfigBean);
model.addAttribute("tableConfig", tableConfig);
return "index";// 与templates中index.html对应
}

@ -1,9 +1,9 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.pojo.request.PlaneRequest;
import com.htfp.weather.web.pojo.request.Position3D;
import com.htfp.weather.web.pojo.request.ProfileRequest;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.*;
import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import org.springframework.validation.annotation.Validated;
@ -11,6 +11,8 @@ import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* @Author : shiyi
@ -28,6 +30,7 @@ public class UpperWeatherController {
@RequestMapping("/profileByPressure")
public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
@ -36,8 +39,8 @@ public class UpperWeatherController {
return Result.success(profileResponse);
}
@PostMapping("/forecast")
public Result queryTimeSeries(@Validated @RequestBody Position3D position3D) {
@PostMapping("/queryForecastTimeSeries")
public Result queryForecastTimeSeries(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
@ -60,12 +63,50 @@ public class UpperWeatherController {
return Result.success(forecastSeries);
}
@PostMapping("/now")
public Result queryNowWeather(@Validated @RequestBody Position3D position3D) {
@PostMapping("/queryUpperNowWeather")
public Result queryUpperNowWeather(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
NowWeatherStatus nowWeather = gfsDataService.getNowWeather(latitude, longitude, level);
return Result.success(nowWeather);
}
@PostMapping("/queryUpperNowWeatherInMultiPoints")
public Result queryUpperNowWeatherInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
Double latitude = position.getLatitude();
Double longitude = position.getLongitude();
if (latitude != null && longitude != null) {
latitudeList.add(latitude);
longitudeList.add(longitude);
} else {
throw new AppExcpetion(ErrorCode.MULTI_INDEX_ERROR);
}
}
int level = multiPointsRequest.getLevel().intValue();
List<NowWeatherStatus> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level);
return Result.success(nowWeatherByMultiPoint);
}
@PostMapping("/queryForecastTimeSeriesInMultiPoints")
public Result queryForecastTimeSeriesInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
Double latitude = position.getLatitude();
Double longitude = position.getLongitude();
if (latitude != null && longitude != null) {
latitudeList.add(latitude);
longitudeList.add(longitude);
} else {
throw new AppExcpetion(ErrorCode.MULTI_INDEX_ERROR);
}
}
int level = multiPointsRequest.getLevel().intValue();
// List<TimeSeriesDataset> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level);
return Result.success();
}
}

@ -8,11 +8,17 @@ package com.htfp.weather.web.exception;
public enum ErrorCode {
//
OTHER_ERROR(-1, "其他错误"),
VALIDATE_ERROR(1001, "参数校验错误 "),
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
DOWNLOAD_ERROR(1004, "数据下载错误"),
DOWNLOAD_ERROR(1005, "数据下载错误"),
TABLESTORE_IMPORT_ERROR(1006, "数据导入TableStore数据库错误"),
MULTI_INDEX_ERROR(1007, "多点索引不正确(null或不在正确范围内)"),
DATA_FILE_OR_DIR_NOT_EXISTS(1008, "目标文件或文件夹不存在"),
CAN_NOT_DELETE_WHEN_IMPORT_OR_DOWNLOAD(1009, "数据导入或下载中,无法进行删除操作"),
// 数据查询
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),
@ -23,10 +29,12 @@ public enum ErrorCode {
DATA_SET_EMPTY(6000, "TableStore数据库为空无法查询"),
QUERY_TIME_ERROR(6001, "目标时间不在当前数据范围内"),
LONGITUDE_INDEX_ERROR(6002, "经度坐标索引"),
LATITUDE_INDEX_ERROR(6003, "纬度坐标索引"),
LEVEL_INDEX_ERROR(6004, "高度坐标索引"),
LONGITUDE_INDEX_ERROR(6002, "经度坐标索引错误"),
LATITUDE_INDEX_ERROR(6003, "纬度坐标索引错误"),
LEVEL_INDEX_ERROR(6004, "高度坐标索引错误"),
INDEX_SIZE_ERROR(6005, "坐标长度不一致"),
NO_SUCH_VARIABLE(6006, "找不到变量名映射"),
RESPONSE_DATA_BUILD_ERROR(6007, "查询数据结果构建错误"),
;
private final int code;
private final String msg;

@ -1,5 +1,6 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.download.GfsDataConfig;
import lombok.Data;
/**
@ -8,7 +9,7 @@ import lombok.Data;
* @Description :
*/
@Data
public class ConfigUpdate {
public class GfsConfigUpdate {
String secret;
GfsDataConfig config;
}

@ -0,0 +1,20 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.griddata.common.TableConfig;
import lombok.Data;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Pattern;
/**
* @Author : shiyi
* @Date : 2024/4/21 18:34
* @Description :
*/
@Data
public class TableConfigUpdate {
@Pattern(regexp = "^htfpweather$", message = "无权限")
String secret;
@NotNull
TableConfig config;
}

@ -0,0 +1,15 @@
package com.htfp.weather.web.pojo.response;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/6/12 11:46
* @Description :
*/
@Data
public class FileInfo {
private String fileName;
private String filePath;
private long fileSize;
}

@ -0,0 +1,9 @@
package com.htfp.weather.web.pojo.response;
/**
* @Author : shiyi
* @Date : 2024/6/6 15:24
* @Description :
*/
public class MultiPointResponse {
}

@ -11,13 +11,15 @@ import lombok.Data;
public class NowWeatherStatus {
String time; // 数据时间,北京时
String weatherText; // 天气状况
float temp; // 温度; // 温度
float windSpeed; // 风速m/s
float wind360; // 风向360角度
Double latitude;
Double longitude;
Float temp; // 温度; // 温度
Float windSpeed; // 风速m/s
Float wind360; // 风向360角度
// String windDir; // 风向, 文字描述,如“西北风”
// String windScale; // 风力等级
float humidity; // 相对湿度 %
// float pressure; // 大气压强,默认单位:百帕
float cloud; // 云量
float precip; //降水量
Float humidity; // 相对湿度 %
// Float pressure; // 大气压强,默认单位:百帕
Float cloud; // 云量
Float precip; //降水量
}

@ -41,6 +41,6 @@ public class Result {
}
public static Result error(Throwable cause) {
return new Result(false, 1001, cause.getMessage(), null);
return new Result(false, null, cause.getMessage(), null);
}
}

@ -14,6 +14,8 @@ import java.util.List;
@Data
public class TimeSeriesDataset {
List<String> time;
Double latitude;
Double longitude;
float[] temp;
float[] windSpeed;
float[] wind360;

@ -0,0 +1,64 @@
package com.htfp.weather.web.service;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.FileInfo;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/6/12 11:25
* @Description :
*/
@Component
public class FileService {
public List<FileInfo> getFileList(File dataDir){
if (!dataDir.exists() || !dataDir.isDirectory()) {
throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS);
}
List<FileInfo> fileInfoList = new ArrayList<>();
File[] files = dataDir.listFiles();
if (files == null || files.length == 0) {
return null;
}
for (File file : files) {
String fileName = file.getName();
if (fileName.endsWith("grib2") || fileName.endsWith("nc")) {
FileInfo fileInfo = new FileInfo();
fileInfo.setFilePath(file.getAbsolutePath());
fileInfo.setFileName(fileName);
fileInfo.setFileSize(file.length());
fileInfoList.add(fileInfo);
}
}
return fileInfoList;
}
public void uploadFile(String filePath){
// 文件上传逻辑
}
public void downloadFile(String filePath){
// 文件下载逻辑
}
public void deleteFile(File file) throws IOException {
if (file == null || !file.exists()) {
throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS);
}
FileUtils.delete(file);
}
public void deleteDir(File dir) throws IOException {
if (!dir.exists() || !dir.isDirectory()) {
throw new AppExcpetion(ErrorCode.DATA_FILE_OR_DIR_NOT_EXISTS);
}
FileUtils.deleteDirectory(dir);
}
}

@ -6,10 +6,11 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import com.htfp.weather.griddata.common.GfsVariableHeightEnum;
import com.htfp.weather.griddata.common.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.QueryMeta;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.NdArrayUtils;
import com.htfp.weather.web.exception.AppExcpetion;
@ -22,8 +23,10 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import ucar.ma2.Array;
import ucar.ma2.Index4D;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.time.Duration;
import java.time.Instant;
import java.time.OffsetDateTime;
@ -37,7 +40,8 @@ import java.util.stream.Collectors;
* @Date : 2024/2/2 15:47
* @Description : tablestore
*/
@Service("tablestore-gfs") @Slf4j
@Service("tablestore-gfs")
@Slf4j
public class GfsDataServiceImpl implements IDataService {
private static final int SURFACE_LEVEL = 9999;
@Resource
@ -45,9 +49,18 @@ public class GfsDataServiceImpl implements IDataService {
@Resource
QueryMeta queryMeta;
@Resource
TableConfigBean tableConfigBean;
TableConfig tableConfig;
/**
* 线
*
* @param targetTime UTC
* @param variableName
* @param latitude
* @param longitude
* @return {@link ProfileResponse}
*/
public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) {
String targetVariable = GfsVariableIsobaricEnum.getGfsVariableName(variableName);
@ -66,7 +79,7 @@ public class GfsDataServiceImpl implements IDataService {
int iLon = getLongitudeIndex(longitude);
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
int[] pressureLevels = tableConfigBean.getPressureList();
int[] pressureLevels = tableConfig.getPressureList();
Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon);
float[] values = (float[]) array.getStorage();
return new ProfileResponse(pressureLevels, values);
@ -76,6 +89,14 @@ public class GfsDataServiceImpl implements IDataService {
}
/**
* 24
*
* @param latitude
* @param longitude
* @param level 9999
* @return @{@link TimeSeriesDataset}
*/
public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
@ -95,6 +116,7 @@ public class GfsDataServiceImpl implements IDataService {
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildTimeSeriesDatasetSurface(timeList, gridDataSet);
} else {
@ -106,12 +128,20 @@ public class GfsDataServiceImpl implements IDataService {
}
}
/**
*
*
* @param latitude
* @param longitude
* @param level 9999
* @return @{@link NowWeatherStatus}
*/
public NowWeatherStatus getNowWeather(double latitude, double longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()");
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage());
}
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta);
int iLat = getLatitudeIndex(latitude);
@ -122,38 +152,106 @@ public class GfsDataServiceImpl implements IDataService {
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
GridDataSet gridDataSet = gfsDataFetcher.getSinglePoint(gridDataSetId, variableNames, iTime, iLev, iLat, iLon);
if (level == SURFACE_LEVEL) {
return buildNowWeatherSurface(gridDataSet);
} else {
return buildNowWeatherPressure(gridDataSet);
}
// 每个变量都是0维数组
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
NowWeatherStatus nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, 0, 0, levelFlag);
nowWeatherStatus.setLatitude(latitude);
nowWeatherStatus.setLongitude(longitude);
return nowWeatherStatus;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private List<String> getVariableNamesInDatabase(int level) {
List<String> variableNames = new ArrayList<>();
if (level == SURFACE_LEVEL) {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
/**
*
* @param latitude
* @param longitude
* @param level
* @return
*/
public List<NowWeatherStatus> getNowWeatherByMultiPoint(List<Double> latitude, List<Double> longitude, int level) {
GridDataSetMeta lastGridDataSetMeta = null;
try {
lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta();
} catch (Exception e) {
throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": " + e.getMessage());
}
if (latitude.size() != longitude.size()) {
throw new AppExcpetion(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致");
}
} else {
for (String e: tableConfigBean.getVariableList()){
String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e);
if (gfsVariableName != null){
variableNames.add(gfsVariableName);
int pointSize = latitude.size();
List<String> variableNames = getVariableNamesInDatabase(level);
int iTime = getTargetTimeIndex(OffsetDateTime.now(), lastGridDataSetMeta);
List<Integer> latIndex = latitude.stream().map(this::getLatitudeIndex).collect(Collectors.toList());
List<Integer> lonIndex = longitude.stream().map(this::getLongitudeIndex).collect(Collectors.toList());
int iMinLat = Collections.min(latIndex).intValue();
int iMaxLat = Collections.max(latIndex).intValue();
int iMinLon = Collections.min(lonIndex).intValue();
int iMaxLon = Collections.max(lonIndex).intValue();
int iLev = getLevelIndex(level);
int latLength = iMaxLat - iMinLat + 1;
int lonLength = iMaxLon - iMinLon + 1;
// int levLength = iMaxLev - iMinLev + 1;
try {
String gridDataSetId = lastGridDataSetMeta.getGridDataSetId();
int[] origin = new int[]{iTime, iLev, 0, 0};
int[] shape = new int[]{1, 1, tableConfig.getLatSize(), tableConfig.getLatSize()};
GridDataSet gridDataSet = gfsDataFetcher.queryByTableStore(gridDataSetId, variableNames,
origin, shape);
List<NowWeatherStatus> nowWeatherStatusList = new ArrayList<>();
for (int i = 0; i < pointSize; i++) {
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
// NOTE 2024/6/11: build中传入的索引和数据库的查询索引意义不同
final GfsLevelsEnum levelFlag = GfsLevelsEnum.getByCode(level);
nowWeatherStatus = buildNowWeatherInTargetPoint(gridDataSet, 0, latIndex.get(i), lonIndex.get(i), levelFlag);
nowWeatherStatus.setLatitude(latitude.get(i));
nowWeatherStatus.setLongitude(longitude.get(i));
nowWeatherStatusList.add(nowWeatherStatus);
}
return nowWeatherStatusList;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return variableNames;
private int[] getMarginIndexByMultiPoint(List<Double> latitude, List<Double> longitude, List<Double> level) {
int minLat = Integer.MIN_VALUE;
int maxLat = Integer.MAX_VALUE;
int minLon = Integer.MIN_VALUE;
int maxLon = Integer.MAX_VALUE;
int minLev = Integer.MIN_VALUE;
int maxLev = Integer.MAX_VALUE;
int iMinLat = getLatitudeIndex(minLat);
int iMaxLat = getLatitudeIndex(maxLat);
int iMinLon = getLongitudeIndex(minLon);
int iMaxLon = getLongitudeIndex(maxLon);
return new int[]{iMinLat, iMaxLat, iMinLon, iMaxLon};
}
/**
*
*
* @param targetTime UTC
* @param variableName
* @param level
* @param minLat
* @param maxLat
* @param minLon
* @param maxLon
* @return @{@link PlaneResponse}
*/
public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) {
int iLev;
String targetVariable;
@ -194,11 +292,32 @@ public class GfsDataServiceImpl implements IDataService {
}
private List<String> getVariableNamesInDatabase(int level) {
List<String> variableNames = new ArrayList<>();
if (level == SURFACE_LEVEL) {
for (String e : tableConfig.getVariableList()) {
String gfsVariableName = GfsVariableHeightEnum.getGfsVariableName(e);
if (gfsVariableName != null) {
variableNames.add(gfsVariableName);
}
}
} else {
for (String e : tableConfig.getVariableList()) {
String gfsVariableName = GfsVariableIsobaricEnum.getGfsVariableName(e);
if (gfsVariableName != null) {
variableNames.add(gfsVariableName);
}
}
}
return variableNames;
}
private PlaneResponse buildPlane(Array array, int iMinLat, int iMinLon, int latLength, int lonLength) {
PlaneResponse planeResponse = new PlaneResponse();
float[][] values = (float[][]) array.reduce().copyToNDJavaArray();
planeResponse.setLatList(Arrays.copyOfRange(tableConfigBean.getLatList(), iMinLat, iMinLat + latLength));
planeResponse.setLonList(Arrays.copyOfRange(tableConfigBean.getLonList(), iMinLon, iMinLon + lonLength));
planeResponse.setLatList(Arrays.copyOfRange(tableConfig.getLatList(), iMinLat, iMinLat + latLength));
planeResponse.setLonList(Arrays.copyOfRange(tableConfig.getLonList(), iMinLon, iMinLon + lonLength));
planeResponse.setValues(values);
return planeResponse;
}
@ -245,49 +364,47 @@ public class GfsDataServiceImpl implements IDataService {
return timeSeriesDataset;
}
private NowWeatherStatus buildNowWeatherPressure(GridDataSet gridDataSet) {
/**
*
* @param gridDataSet
* @param iLevInGrid-
* @param iLatInGrid-
* @param iLonInGrid-
* @return
*/
private NowWeatherStatus buildNowWeatherInTargetPoint(GridDataSet gridDataSet, int iLevInGrid, int iLatInGrid, int iLonInGrid, GfsLevelsEnum levelFlag) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
NowWeatherStatus result = new NowWeatherStatus();
result.setTime(timeStr);
// 利用反射构建数据集
Class<? extends NowWeatherStatus> aClass = result.getClass();
for (Map.Entry<String, Grid4D> entry : gridDataSet.getVariables().entrySet()) {
String variableNameInFile = entry.getKey();
String variableNameInApi;
if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {
variableNameInApi = GfsVariableHeightEnum.getVariableNameInApi(variableNameInFile);
} else {
variableNameInApi = GfsVariableIsobaricEnum.getVariableNameInApi(variableNameInFile);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
if (variableNameInApi == null) {
throw new AppExcpetion(ErrorCode.NO_SUCH_VARIABLE);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
Grid4D grid4D = entry.getValue(); Index4D index = new Index4D(grid4D.getShape());
try {
String setMethodName = "set" + variableNameInApi.substring(0, 1).toUpperCase() + variableNameInApi.substring(1);
Method method = aClass.getDeclaredMethod(setMethodName, aClass.getDeclaredField(variableNameInApi).getType());
method.invoke(result, grid4D.toArray().getFloat(index.set(0, iLevInGrid, iLatInGrid, iLonInGrid)));
} catch (ReflectiveOperationException e) {
throw new AppExcpetion(ErrorCode.NO_SUCH_VARIABLE, e);
} catch (Exception e) {
e.printStackTrace();
throw new AppExcpetion(ErrorCode.RESPONSE_DATA_BUILD_ERROR);
}
grid4D = gridDataSet.getVariable(GfsVariableIsobaricEnum.getGfsVariableName(GfsVariableIsobaricEnum.CLOUD));
if (grid4D != null) {
nowWeatherStatus.setCloud(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
return result;
}
private NowWeatherStatus buildNowWeatherSurface(GridDataSet gridDataSet) {
String timeStr = OffsetDateTime.now().withMinute(0).withNano(0).format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING));
NowWeatherStatus nowWeatherStatus = new NowWeatherStatus();
nowWeatherStatus.setTime(timeStr);
Grid4D grid4D;
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.TEMP));
if (grid4D != null) {
nowWeatherStatus.setTemp(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND_SPEED));
if (grid4D != null) {
nowWeatherStatus.setWindSpeed(((float[]) grid4D.toArray().getStorage())[0]);
}
grid4D = gridDataSet.getVariable(GfsVariableHeightEnum.getGfsVariableName(GfsVariableHeightEnum.WIND360));
if (grid4D != null) {
nowWeatherStatus.setWind360(((float[]) grid4D.toArray().getStorage())[0]);
}
return nowWeatherStatus;
}
private int getTargetTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) {
long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName());
OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0));
@ -336,8 +453,9 @@ public class GfsDataServiceImpl implements IDataService {
map.put("timeList", timeList);
return map;
}
private int getLongitudeIndex(double lon) {
double[] lonList = tableConfigBean.getLonList();
double[] lonList = tableConfig.getLonList();
if (lonList.length == 0) {
throw new AppExcpetion(ErrorCode.LONGITUDE_INDEX_ERROR, "经度列表为空, 无法查找索引");
}
@ -354,7 +472,7 @@ public class GfsDataServiceImpl implements IDataService {
}
private int getLatitudeIndex(double lat) {
double[] latList = tableConfigBean.getLatList();
double[] latList = tableConfig.getLatList();
if (latList.length == 0) {
throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "纬度列表为空, 无法查找索引");
}
@ -372,7 +490,7 @@ public class GfsDataServiceImpl implements IDataService {
}
private int getPressureIndex(int pressureLevel) {
List<Integer> levList = Arrays.stream(tableConfigBean.getPressureList()).boxed().collect(Collectors.toList());
List<Integer> levList = Arrays.stream(tableConfig.getPressureList()).boxed().collect(Collectors.toList());
if (CollectionUtils.isEmpty(levList)) {
throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "气压列表为空, 无法查找索引");
}

@ -107,7 +107,11 @@ public class CaiYunServiceImpl implements ISurfaceDataService {
HashMap<String, String> params = new HashMap<>();
params.put("alert", "true");
CaiYunWarningResponse response = caiYunRequest(url, params, CaiYunWarningResponse.class);
return response.getResult().getAlert().getContent();
CaiYunWarningResponse.AlertResult.Alert alert = response.getResult().getAlert();
if (alert == null) {
throw new AppExcpetion(ErrorCode.CAI_YUN_REQUEST_ERROR, ":无可用预警数据");
}
return alert.getContent();
}
private NowWeatherStatus buildNowSurfaceWeatherStatus(CaiYunNowResponse nowResponse) {

@ -5,7 +5,7 @@
"minLat" : 0.0,
"maxLat" : 55.0,
"resolution" : 0.25,
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST"],
"pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000],
"saveRoot" : "GFSData"
"variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ,"GUST", "PRATE"],
"pressureLevels" : [400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000],
"saveRoot" : "./GFSData"
}

@ -36,10 +36,8 @@ class GfsDownloaderTest {
void downloadAll() {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
boolean isComplete = gfsDownloader.downloadAll(fileInfoList);
if (isComplete) {
System.out.println("下载完成");
}
gfsDownloader.downloadAll(fileInfoList);
}
@Test

@ -19,9 +19,12 @@ class GfsUtilsTest {
@Test
void getRefTime() throws IOException {
NetcdfFile ncFile = NetcdfFiles.open("C:/Users/shi_y/Desktop/GFSData/UTC-20230905/BJT-20230905-0900.grib2");
String filename = "D:\\HTFP\\weather\\GFSData\\UTC-20240607.00\\BJT-20240607.17-from-UTC-20240607.00+9.grib2";
NetcdfFile ncFile = NetcdfFiles.open(filename);
String refTime = GfsUtils.getRefTime(ncFile);
assertEquals("2023-09-05T00:00+00:00", refTime);
Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN));
String s = filename.split("\\+")[1];
Integer.parseInt(s.split(".grib2")[0]);
// assertEquals("2023-09-05T00:00+00:00", refTime);
// Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN));
}
}

@ -5,8 +5,6 @@ import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/5/17 11:44
@ -15,9 +13,9 @@ import static org.junit.jupiter.api.Assertions.*;
@SpringBootTest
class GridDataProcesserTest {
@Resource
GridDataProcesser gridDataProcesser;
GridDataProcessor gridDataProcessor;
@Test
void dailyDataProcess() {
gridDataProcesser.clearExpiredData();
gridDataProcessor.clearExpiredData();
}
}

@ -0,0 +1,24 @@
package com.htfp.weather.web.service;
import com.htfp.weather.web.pojo.response.FileInfo;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/6/12 11:52
* @Description :
*/
class FileServiceTest {
FileService fileService = new FileService();
@Test
void getFileList() {
File dir = new File("D:\\HTFP\\weather\\weather-service\\GFSData\\UTC-20240611.00");
List<FileInfo> fileList = fileService.getFileList(dir);
System.out.println(fileList);
}
}

@ -0,0 +1,12 @@
{
"dataTableName" : "gfs_data_table",
"metaTableName" : "gfs_meta_table",
"dataIndexName" : "gfs_data_index",
"metaIndexName" : "gfs_meta_table_index",
"dataDir" : "./GFSData",
"variableList" : [ "temp", "cloud", "wind360", "windSpeed"],
"lonSize" : 281,
"latSize" : 221,
"levSize" : 10,
"timeToLive" : 2
}
Loading…
Cancel
Save