包名更换;api增加版本号

download
shiyi 3 months ago
parent 11d010ec1b
commit b92795b619

2
.gitignore vendored

@ -21,3 +21,5 @@
*.iws
*.iml
*.ipr
*.json

@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<!--父pom.xml-->
<groupId>com.htfp</groupId>
<artifactId>weather</artifactId>
<version>0.0.1</version>
<packaging>pom</packaging>
<name>weather app</name>
<description>weather service prepublish</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.13</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<!--声明子模块-->
<modules>
<module>tablestore-grid-master</module>
<module>weather-service</module>
</modules>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<configuration>
<skipTests>false</skipTests> <!--默认关掉单元测试 -->
</configuration>
</plugin>
</plugins>
</build>
</project>

@ -7,6 +7,20 @@
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.1-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
<groupId>com.htfp</groupId>
<artifactId>weather</artifactId>
<version>0.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<repositories>
<repository>
@ -113,10 +127,15 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -1,24 +0,0 @@
package com.aliyun.tablestore.example.grid.common;
import ucar.ma2.DataType;
public class ExampleConfig {
/**
* index
*/
public static String GRID_DATA_TABLE_NAME = "GRID_DATA_TABLE_EXAMPLE1";
public static String GRID_META_TABLE_NAME = "GRID_META_TABLE_EXAMPLE1";
public static String GRID_META_INDEX_NAME = "GRID_META_INDEX";
/**
* ID
*/
public static String EXAMPLE_GRID_DATA_SET_ID = "test_echam_spectral_example";
public static String EXAMPLE_GRID_DATA_SET_NC_FILE_PATH = "test_echam_spectral.nc";
public static String EXAMPLE_GRID_DATA_VARIABLE = "tpot";
public static int[] EXAMPLE_GRID_DATA_SHAPE = new int[]{8, 47, 96, 192};
public static DataType EXAMPLE_GRID_DATA_TYPE = DataType.FLOAT;
}

@ -1,62 +0,0 @@
package com.aliyun.tablestore.example.grid.common;
import com.google.gson.Gson;
import org.apache.commons.io.IOUtils;
import java.io.FileInputStream;
import java.io.InputStream;
public class TableStoreConf {
private String endpoint;
private String accessId;
private String accessKey;
private String instanceName;
/**
* instance
*/
public static TableStoreConf newInstance(String path) {
try {
InputStream f = new FileInputStream(path);
Gson gson = new Gson();
return gson.fromJson(IOUtils.toString(f), TableStoreConf.class);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getAccessId() {
return accessId;
}
public void setAccessId(String accessId) {
this.accessId = accessId;
}
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getInstanceName() {
return instanceName;
}
public void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
}

@ -1,33 +0,0 @@
package com.aliyun.tablestore.example.grid.common;
import com.aliyun.tablestore.grid.TableStoreGrid;
import com.aliyun.tablestore.grid.TableStoreGridConfig;
public abstract class TableStoreGridExample {
protected TableStoreGrid tableStoreGrid;
// private String pathSeperator = "/";
public TableStoreGridExample(String dataTableName, String metaTableName) {
// String os = System.getProperty("os.name");
// if (os.toLowerCase().startsWith("win")) {
// pathSeparator = "\\";
// }
String pathSeparator = System.getProperty("file.separator");
TableStoreConf conf = TableStoreConf.newInstance(System.getProperty("user.dir") + pathSeparator + "tablestoreConf.json");
TableStoreGridConfig config = new TableStoreGridConfig();
config.setTableStoreEndpoint(conf.getEndpoint());
config.setAccessId(conf.getAccessId());
config.setAccessKey(conf.getAccessKey());
config.setTableStoreInstance(conf.getInstanceName());
config.setDataTableName(dataTableName);
config.setMetaTableName(metaTableName);
tableStoreGrid = new TableStoreGrid(config);
}
public void close() {
if (tableStoreGrid != null) {
tableStoreGrid.close();
}
}
}

@ -1,5 +1,7 @@
package com.aliyun.tablestore.grid.consts;
import com.alicloud.openservices.tablestore.model.search.FieldType;
import java.util.function.Consumer;
/**
@ -9,14 +11,14 @@ import java.util.function.Consumer;
*/
public enum AttributionEnum {
//
STATUS("status", "导入状态"),
CREATE_TIME("create_time", "创建时间"),
REFERENCE_TIME("reference_time", "起报时间"),
STATUS("status", "导入状态", FieldType.KEYWORD),
CREATE_TIME("create_time", "创建时间", FieldType.LONG),
REFERENCE_TIME("reference_time", "起报时间", FieldType.LONG),
;
public String name;
public String info;
public FieldType type;
public String getName() {
return name;
}
@ -24,8 +26,13 @@ public enum AttributionEnum {
public String getInfo() {
return info;
}
AttributionEnum(String name, String info) {
public FieldType getType() {
return type;
}
AttributionEnum(String name, String info, FieldType type) {
this.name = name;
this.info = info;
this.type = type;
}
}

@ -91,6 +91,7 @@ public class TableStoreDataWriter implements GridDataWriter {
if (meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) {
List<Column> columns = splitDataToColumns(grid2D);
writeToTableStore(variable, t, z, columns);
columns = null;
} else {
throw new IllegalArgumentException("unsupported store type");
}

@ -52,7 +52,6 @@ public abstract class Grid {
}
public Array toArray() {
Array array = Array.factory(dataType, shape, buffer.duplicate());
return array;
return Array.factory(dataType, shape, buffer.duplicate());
}
}

@ -1,4 +1,7 @@
import com.aliyun.tablestore.grid.model.grid.Grid;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.ma2.Index;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.NetcdfFile;
@ -6,6 +9,7 @@ import ucar.nc2.NetcdfFiles;
import ucar.nc2.Variable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@ -13,7 +17,15 @@ import java.time.format.DateTimeFormatter;
* Unit test for simple App.
*/
public class TestReadGrib2ByNC {
public static void main(String[] args){
public static void main(String[] args) throws InterruptedException {
// while (true) {
// ByteBuffer allocate = ByteBuffer.allocate(500 * 500 * 4);
// Grid2D grid = new Grid2D(allocate, DataType.FLOAT, new int[]{0, 0}, new int[]{500,500});
// allocate = null; grid=null;
// Thread.sleep(300);
// }
}
public static void main1(String[] args){
// String ncFile = "surface.nc";
String file = "C:\\Users\\shi_y\\Desktop\\java_learn\\data_download\\GFSData\\UTC-20230910\\BJT-20230911-0200.grib2";
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {

@ -42,7 +42,7 @@ public class CreateStoreExample extends TableStoreGridExample {
this.tableStoreGrid.createMetaIndex(ExampleConfig.GRID_META_INDEX_NAME, indexSchema);
}
public static void main(String[] args) throws Exception {
public static void test(String[] args) throws Exception {
CreateStoreExample example = new CreateStoreExample();
try {
example.createStore();

@ -6,7 +6,6 @@ import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import ucar.ma2.Array;
import java.util.Arrays;
import java.util.Collections;
public class DataFetchExample extends TableStoreGridExample {
@ -71,7 +70,7 @@ public class DataFetchExample extends TableStoreGridExample {
fetch3();
}
public static void main(String[] args) throws Exception {
public static void test(String[] args) throws Exception {
DataFetchExample example = new DataFetchExample();
try {
example.run();

@ -12,7 +12,6 @@ import ucar.ma2.DataType;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -97,7 +96,7 @@ public class DataImportExample extends TableStoreGridExample {
updateMeta(meta);
}
public static void main(String[] args) throws Exception {
public static void test(String[] args) throws Exception {
DataImportExample example = new DataImportExample();
try {
log.info("导入数据");

@ -11,7 +11,6 @@ import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import lombok.extern.slf4j.Slf4j;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

@ -2,11 +2,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.htfp.weather</groupId>
<groupId>com.htfp</groupId>
<artifactId>weather-service</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.1</version>
<name>weather-service</name>
<description>weather-service</description>
<packaging>jar</packaging>
<!--声明父模块-->
<parent>
<groupId>com.htfp</groupId>
<artifactId>weather</artifactId>
<version>0.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@ -88,11 +97,10 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<!--? <dependency>-->
<!--? <groupId>org.springframework.boot</groupId>-->
<!--? <artifactId>spring-boot-starter-thymeleaf</artifactId>-->
<!--? </dependency>-->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
@ -128,48 +136,34 @@
</dependency>
<!-- springmvc的参数valid校验依赖 结束 -->
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<executable>true</executable>
<layout>JAR</layout>
<!-- 指定该Main Class为全局的唯一入口 -->
<mainClass>com.htfp.weather.WeatherServiceApplication</mainClass>
<layout>ZIP</layout>
</configuration>
<executions>
<execution>
<id>repackage</id>
<goals>
<!--可以把依赖的包都打包到生成的Jar包中-->
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

@ -26,6 +26,7 @@ public class DownLoadFileInfo {
private String savePath;
private boolean isDownloadSuccess;
private String errorMsg;
public DownLoadFileInfo() {
}

@ -108,7 +108,7 @@ public class GfsDataConfig {
}
public void readConfig() {
try (InputStream f = new FileInputStream(configPath)){
GfsDataConfig gfsDataConfig = JSONUtils.json2pojo(IOUtils.toString(f), GfsDataConfig.class);
GfsDataConfig gfsDataConfig = JSONUtils.json2obj(IOUtils.toString(f), GfsDataConfig.class);
this.duration = gfsDataConfig.getDuration();
this.minLon = gfsDataConfig.getMinLon();
this.maxLon = gfsDataConfig.getMaxLon();

@ -75,8 +75,13 @@ public class GfsDownloader extends BaseDataDownloader {
try {
// DONE: 改用okhttp FIXME 2024/6/8: 如果连接是httpsjar包启动下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
HttpClientUtils.downloadFileByUrl(url, fileOut.getPath());
boolean fileIsValid = fileValid(fileOut.getAbsolutePath());
downLoadFileInfo.setDownloadSuccess(fileIsValid);
if (fileIsValid) {
log.info("[GFS Download] 文件下载成功: {}", fileOut);
downLoadFileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} else {
log.error("[GFS Download] 文件下载失败,文件打开错误: {}", fileOut);
}
} catch (Exception e) {
// DONE 2024/5/24: 文件服务器在外网不稳定,增加重试机制
downLoadFileInfo.setDownloadSuccess(false);
@ -85,7 +90,8 @@ public class GfsDownloader extends BaseDataDownloader {
return download0(downLoadFileInfo, retryNum - 1);
} else {
e.printStackTrace();
log.error("[GFS Download] 文件下载失败: {}", fileOut);
downLoadFileInfo.setErrorMsg(e.getMessage());
log.error("[GFS Download] 文件{}下载失败: {}", fileOut, e.getMessage());
}
}
return downLoadFileInfo;
@ -97,6 +103,7 @@ public class GfsDownloader extends BaseDataDownloader {
ncFile.getLocation();
return true;
} catch (IOException e){
log.error("GFS文件{}校验失败: {}", file, e.getMessage());
return false;
}
}

@ -81,7 +81,7 @@ public class TableConfig {
// String pathSeparator = System.getProperty("file.separator");
try (InputStream f = new FileInputStream(configPath)){
String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8);
TableConfig tableConfig = JSONUtils.json2pojo(jsonStr, TableConfig.class);
TableConfig tableConfig = JSONUtils.json2obj(jsonStr, TableConfig.class);
this.dataTableName = tableConfig.getDataTableName();
this.metaTableName = tableConfig.getMetaTableName();
this.dataIndexName = tableConfig.getDataIndexName();

@ -1,6 +1,5 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import org.apache.commons.io.IOUtils;
@ -25,7 +24,7 @@ public class TableStoreConf {
String path = System.getProperty("user.dir") + "./tablestoreConf.json";
InputStream f = new FileInputStream(path);
String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8);
return JSONUtils.json2pojo(jsonStr, TableStoreConf.class);
return JSONUtils.json2obj(jsonStr, TableStoreConf.class);
} catch (Exception ex) {
throw new RuntimeException(ex);
}

@ -19,10 +19,11 @@ public abstract class BaseTableOperation{
private TableConfig tableConfig;
@Autowired
private TableStoreConf tableStoreConf;
TableStoreGridConfig config;
@PostConstruct
public void init() {
TableStoreGridConfig config = new TableStoreGridConfig();
config = new TableStoreGridConfig();
if (tableStoreConf == null){
tableStoreConf = TableStoreConf.init();
}
@ -36,12 +37,25 @@ public abstract class BaseTableOperation{
config.setTableStoreInstance(tableStoreConf.getInstanceName());
config.setDataTableName(tableConfig.getDataTableName());
config.setMetaTableName(tableConfig.getMetaTableName());
connect();
}
/**
*
*/
public void connect() {
if (tableStoreGrid == null) {
tableStoreGrid = new TableStoreGrid(config);
}
}
/**
*
*/
public void close() {
if (tableStoreGrid != null) {
tableStoreGrid.close();
tableStoreGrid = null;
}
}
}

@ -39,7 +39,7 @@ public class CreateTable extends BaseTableOperation {
AttributionEnum[] values = AttributionEnum.values();
ArrayList<FieldSchema> fieldSchemas = new ArrayList<>();
for (AttributionEnum attributionEnum : values) {
fieldSchemas.add(new FieldSchema(attributionEnum.getName(), FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true));
fieldSchemas.add(new FieldSchema(attributionEnum.getName(), attributionEnum.getType()).setIndex(true).setEnableSortAndAgg(true));
}
indexSchema.setFieldSchemas(fieldSchemas);
this.tableStoreGrid.createMetaIndex(TableConfigStatic.META_INDEX_NAME, indexSchema);

@ -20,6 +20,7 @@ import org.springframework.util.CollectionUtils;
import ucar.ma2.Array;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import java.time.Duration;
import java.time.Instant;
@ -43,7 +44,10 @@ public class GfsDataFetcher extends BaseTableOperation {
@Resource
QueryMeta queryMeta;
GridDataSetMeta lastGridDataSetMeta;
@PreDestroy
public void destroy() {
close();
}
public GridDataSetMeta getLastGridDataSetMeta() {
if (lastGridDataSetMeta == null) {
updateLastGridDataSetMeta();

@ -3,6 +3,7 @@ package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.GridDataWriter;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
@ -17,9 +18,9 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.DependsOn;
import org.springframework.context.annotation.Description;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import sun.nio.ch.DirectBuffer;
import ucar.ma2.*;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
@ -27,8 +28,8 @@ import ucar.nc2.Variable;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import javax.annotation.security.DenyAll;
import java.io.File;
import java.nio.ByteBuffer;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -50,7 +51,7 @@ public class GfsDataImport extends BaseTableOperation {
@Getter
boolean importing = false;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
GridDataWriter writer;
/**
*
*
@ -91,6 +92,7 @@ public class GfsDataImport extends BaseTableOperation {
/** 导入指定起报时间的所有文件*/
public synchronized List<ImportFileInfo> importData(OffsetDateTime refTime) throws Exception {
importing = true;
connect();
try {
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
@ -103,8 +105,9 @@ public class GfsDataImport extends BaseTableOperation {
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
writer = tableStoreGrid.getDataWriter(meta);
List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
// todo 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
List<ImportFileInfo> finishedList;
// List<Future<ImportResult>> futures = new ArrayList<>();
@ -138,7 +141,7 @@ public class GfsDataImport extends BaseTableOperation {
meta.addAttribute("status", "DONE");
}
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start) / 1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
@ -147,6 +150,7 @@ public class GfsDataImport extends BaseTableOperation {
return finishedList;
} finally {
importing = false;
close();
}
}
@ -159,8 +163,6 @@ public class GfsDataImport extends BaseTableOperation {
*/
public ImportFileInfo importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
// 相对reftime的小时数文件可能缺失因此time可能是不连续的但是iTime是连续的
if (forecastHour == -1) {
forecastHour = (int) ncFile.findVariable("time").read().getDouble(0);
@ -182,36 +184,47 @@ public class GfsDataImport extends BaseTableOperation {
int zSize = variable.getShape(1);
for (int z = 0; z < zSize; z++) {
// 注意高度索引是递增排序
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
ByteBuffer dataBuffer = transferUnit(
variableName, variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize})
).getDataAsByteBuffer();
Grid2D grid2D = new Grid2D(dataBuffer, variable.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, z, grid2D);
dataBuffer=null; grid2D = null;
}
} else if (shapeLength == 3) {
// DONE 2024/5/24: 导入不含高度坐标的数据,如降水量,阵风风速
Array array = variable.read(new int[]{0, 0, 0}, new int[]{1,xsize, ysize});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
ByteBuffer dataBuffer = transferUnit(
variableName, variable.read(new int[]{0, 0, 0}, new int[]{1,xsize, ysize}))
.getDataAsByteBuffer();
Grid2D grid2D = new Grid2D(dataBuffer, variable.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D(variableName, iTime, 0, grid2D);
dataBuffer= null; grid2D = null;
}
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
}
// 导入风速风向
importWind(writer, meta, ncFile, iTime);
importWind(meta, ncFile, iTime);
} catch (Exception e) {
log.error("[tablestore] 导入文件数据失败,_t={}: {}", forecastHour, file, e);
return new ImportFileInfo(false, file, forecastHour, iTime);
} finally {
System.gc();
}
log.info("[tablestore] 导入文件数据成功,_t={}: {}", forecastHour, file);
return new ImportFileInfo(true, file, forecastHour, iTime);
}
/**单位转换*/
private void transferUnit(String variableName, Array array) {
/**
*
*
* @return
*/
private Array transferUnit(String variableName, Array array) {
if (GfsVariableIsobaricEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
@ -221,10 +234,11 @@ public class GfsDataImport extends BaseTableOperation {
if (GfsVariableHeightEnum.PRECIP.getNameInFile().equals(variableName)) {
MeteoUtils.precipRate2mmPerHour(array);
}
return array;
}
/** 计算风速风向,包括地面坐标和气压坐标,并导入数据库*/
private void importWind(GridDataWriter writer, GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
private void importWind( GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向最好单独保存为一个文件,如果后续需要二维可视化,可以直接本地读取出图,减少网络请求
for (String suffix : new String[]{"_isobaric", "_height_above_ground"}) {
Variable uwnd = ncFile.findVariable("u-component_of_wind" + suffix);
@ -248,6 +262,9 @@ public class GfsDataImport extends BaseTableOperation {
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction" + suffix, iTime, z, dirGrid2D);
uwndArray = null; vwndArray = null; speedArray = null; dirArray = null;
speedGrid2D = null; dirGrid2D = null;
}
}
}

@ -3,6 +3,7 @@ package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.model.search.sort.FieldSort;
import com.alicloud.openservices.tablestore.model.search.sort.Sort;
import com.alicloud.openservices.tablestore.model.search.sort.SortOrder;
import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.core.QueryBuilder;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
@ -11,7 +12,10 @@ import com.htfp.weather.griddata.common.TableConfig;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import javax.crypto.interfaces.PBEKey;
import java.util.Collections;
import java.util.List;
@ -24,6 +28,11 @@ import java.util.List;
public class QueryMeta extends BaseTableOperation {
@Resource
TableConfig tableConfig;
@PreDestroy
public void destroy() {
close();
}
/**
*
*
@ -37,7 +46,8 @@ public class QueryMeta extends BaseTableOperation {
QueryBuilder.and()
.equal("status", "DONE")
.build(),
new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("reference_time", SortOrder.DESC)))));
new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort(AttributionEnum.REFERENCE_TIME.getName(), SortOrder.DESC))))
);
List<GridDataSetMeta> gridDataSetMetas = result.getGridDataSetMetas();
if (CollectionUtils.isEmpty(gridDataSetMetas)) {
throw new RuntimeException("meta table为空");

@ -50,8 +50,8 @@ public class GridDataProcessor {
private final ExecutorService executorService = Executors.newFixedThreadPool(3);
boolean test = false;
/**每日北京时0,8,12,18点执行*/
@Scheduled(cron = "0 0 0,8,12,18 * * ?")
/**每日北京时0,8,12,18点 30分执行*/
@Scheduled(cron = "0 30 0,8,12,18 * * ?")
public void dailyDataProcess() {
if (download()) {
importToTableStore();

@ -53,7 +53,7 @@ public class HttpClientUtils {
// }
public static <T> T sendGet(String url, Map<String, String> params, Class<T> responseClass) throws Exception {
String jsonStr = sendGet(url, params);
return JSONUtils.json2pojo(jsonStr, responseClass);
return JSONUtils.json2obj(jsonStr, responseClass);
}
public static String sendGet(String url, Map<String, String> params) throws IOException {
StringBuilder stringBuilder = new StringBuilder();
@ -92,7 +92,7 @@ public class HttpClientUtils {
}
public static boolean downloadFileByUrl(String sourceUrl, String localPath) {
public static void downloadFileByUrl(String sourceUrl, String localPath) throws IOException {
File file = new File(localPath);
Request request = new Request.Builder()
.url(sourceUrl)
@ -100,15 +100,15 @@ public class HttpClientUtils {
try (Response response = DONLOAD_OKHTTP_CLIENT.newCall(request).execute()) {
if (response.code() == 200) {
assert response.body() != null;
InputStream stream = response.body().byteStream();
Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
long dataBytes = Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
log.debug("文件{}下载完成,文件大小: {} bytes", localPath, dataBytes);
} else {
return false;
throw new RuntimeException("文件下载失败Response" + response);
}
} catch (IOException e) {
return false;
}
return true;
}
}

@ -1,13 +1,13 @@
package com.htfp.weather.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -20,67 +20,58 @@ public class JSONUtils {
}
public static ObjectMapper getInstance() {
return objectMapper;
}
/**
* javaBean,list,array convert to json string
*/
public static String obj2json(Object obj) throws Exception {
public static String obj2json(Object obj) throws JsonProcessingException {
return objectMapper.writeValueAsString(obj);
}
/**
* json string convert to javaBean
*/
public static <T> T json2pojo(String jsonStr, Class<T> clazz)
throws Exception {
public static <T> T json2obj(String jsonStr, Class<T> clazz) throws IOException {
return objectMapper.readValue(jsonStr, clazz);
}
/**
* JSON, T
*/
public static <T> T json2obj(String json, TypeReference<T> typeReference) throws IOException {
return objectMapper.readValue(json, typeReference);
}
/**
* json string convert to map
*/
public static <T> Map<String, Object> json2map(String jsonStr)
throws Exception {
return objectMapper.readValue(jsonStr, Map.class);
public static Map<String, Object> json2map(String jsonStr) throws IOException {
return objectMapper.readValue(jsonStr, new TypeReference<Map<String, Object>>() {});
}
/**
* json string convert to map with javaBean
* JSON string convert to map with specific javaBean
*/
public static <T> Map<String, T> json2map(String jsonStr, Class<T> clazz)
throws Exception {
Map<String, Map<String, Object>> map = (Map<String, Map<String, Object>>) objectMapper.readValue(jsonStr,
new TypeReference<Map<String, T>>() {
});
Map<String, T> result = new HashMap<String, T>();
for (Map.Entry<String, Map<String, Object>> entry : map.entrySet()) {
result.put(entry.getKey(), map2pojo(entry.getValue(), clazz));
}
return result;
public static <T> Map<String, T> json2map(String jsonStr, Class<T> clazz) throws IOException {
JavaType type = objectMapper.getTypeFactory().constructMapType(Map.class, String.class, clazz);
return objectMapper.readValue(jsonStr, type);
}
/**
* json array string convert to list with javaBean
* json array string convert to list with specific javaBean
*/
public static <T> List<T> json2list(String jsonArrayStr, Class<T> clazz)
throws Exception {
List<Map<String, Object>> list = (List<Map<String, Object>>) objectMapper.readValue(jsonArrayStr,
new TypeReference<List<T>>() {
});
List<T> result = new ArrayList<T>();
for (Map<String, Object> map : list) {
result.add(map2pojo(map, clazz));
}
return result;
throws IOException {
return objectMapper.readValue(
jsonArrayStr,
objectMapper.getTypeFactory().constructCollectionType(List.class, clazz)
);
}
/**
* map convert to javaBean
*/
public static <T> T map2pojo(Map map, Class<T> clazz) {
public static <T> T map2obj(Map<String, T> map, Class<T> clazz) {
return objectMapper.convertValue(map, clazz);
}
@ -89,4 +80,7 @@ public class JSONUtils {
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
objectMapper.writeValue(new File(filePath), obj);
}
public static void main(String[] args) throws Exception {
}
}

@ -1,6 +1,6 @@
package com.htfp.weather.web.config.aspect;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.param.response.Result;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.JoinPoint;

@ -1,17 +1,22 @@
package com.htfp.weather.web.config.interceptor;
import com.htfp.weather.utils.JSONUtils;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.param.response.Result;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
import javax.crypto.Mac;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Formatter;
import java.util.List;
/**
* @Author : shiyi
@ -27,10 +32,12 @@ public class InterfaceInterceptor implements HandlerInterceptor {
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
String servletPath = request.getServletPath();
// log.info("InterfaceInterceptor.preHandle >> 进入拦截, {}, {}", servletPath, handler.getClass().getName());
// ip 校验
if (checkIp(request)) {
return true;
}
// 签名校验
final String signature = request.getHeader("signature");
final String timestamp = String.valueOf(request.getHeader("timestamp"));
@ -38,6 +45,7 @@ public class InterfaceInterceptor implements HandlerInterceptor {
String checkSignature = createSignature(timestamp, requestURI);
if (!checkSignature.equalsIgnoreCase(signature)) {
failedResponse(response);
return false;
}
@ -45,25 +53,25 @@ public class InterfaceInterceptor implements HandlerInterceptor {
long nowTime = System.currentTimeMillis();
long reqTime = Long.parseLong(timestamp);
if (Math.abs(nowTime - reqTime) > SECONDS) {
failedResponse(response);
return false;
}
return true;
}
private void failedResponse(HttpServletResponse response) throws IOException {
response.getWriter().write(JSONUtils.obj2json(Result.error(ErrorCode.API_VALIDATE_ERROR)));
}
private boolean checkIp(HttpServletRequest request) {
String remoteAddr = request.getRemoteAddr();
if ("127.0.0.1".equals(remoteAddr) || "0:0:0:0:0:0:0:1".equals(remoteAddr)) {
List<String> whiteNameList = Arrays.asList("127.0.0.1", "182.92.130.23", "123.57.5.1", "0:0:0:0:0:0:0:1");
if (whiteNameList.contains(remoteAddr)) {
return true;
}
if (remoteAddr.startsWith("172.10.0.")) {
// 局域网测试
return true;
}
if ("182.92.130.23".equals(remoteAddr)) {
// 本地服务
return true;
}
return false;
}

@ -11,13 +11,13 @@ import com.htfp.weather.web.config.Config;
import com.htfp.weather.web.config.aspect.ControllerLog;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.GfsConfigUpdate;
import com.htfp.weather.web.pojo.request.TableConfigUpdate;
import com.htfp.weather.web.pojo.response.DownloadResult;
import com.htfp.weather.web.pojo.response.ImportResult;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.param.request.GfsConfigUpdate;
import com.htfp.weather.web.param.request.TableConfigUpdate;
import com.htfp.weather.web.param.response.DownloadResult;
import com.htfp.weather.web.param.response.ImportResult;
import com.htfp.weather.web.param.response.Result;
import com.htfp.weather.web.service.FileService;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import com.htfp.weather.web.service.GfsUpperDataServiceImpl;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.util.CollectionUtils;
@ -52,7 +52,7 @@ public class ConfigController {
@Resource
FileService fileService;
@Resource
GfsDataServiceImpl gfsDataService;
GfsUpperDataServiceImpl gfsDataService;
@Resource
SurfaceWeatherController surfaceWeatherController;
@ -229,7 +229,7 @@ public class ConfigController {
File dataDir = new File(gfsDataConfig.getSaveRoot(), timeStr);
// 查找目录下的文件
List<com.htfp.weather.web.pojo.response.FileInfo> fileList = fileService.getFileList(dataDir);
List<com.htfp.weather.web.param.response.FileInfo> fileList = fileService.getFileList(dataDir);
return Result.success(fileList);
}

@ -2,7 +2,7 @@ package com.htfp.weather.web.controller;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.param.response.Result;
import org.springframework.validation.BindException;
import org.springframework.validation.ObjectError;
import org.springframework.web.bind.annotation.ExceptionHandler;

@ -2,12 +2,13 @@ package com.htfp.weather.web.controller;
import com.htfp.weather.web.config.aspect.ControllerLog;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.Position2D;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import com.htfp.weather.web.param.cma.CmaWarning;
import com.htfp.weather.web.param.request.Position2D;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.Result;
import com.htfp.weather.web.param.response.SurfaceWeatherWarning;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import com.htfp.weather.web.service.GfsUpperDataServiceImpl;
import com.htfp.weather.web.service.surfaceapi.CaiYunServiceImpl;
import com.htfp.weather.web.service.surfaceapi.CmaServiceImpl;
import com.htfp.weather.web.service.surfaceapi.HeFengServiceImpl;
@ -28,7 +29,7 @@ import java.util.List;
@CrossOrigin
@RestController
@Slf4j
@RequestMapping("/htfp/weather/surface/")
@RequestMapping("/htfp/weather/v1/surface/")
public class SurfaceWeatherController {
@Resource
CmaServiceImpl cmaService;
@ -37,7 +38,7 @@ public class SurfaceWeatherController {
@Resource
CaiYunServiceImpl caiYunService;
@Resource
GfsDataServiceImpl gfsDataService;
GfsUpperDataServiceImpl gfsDataService;
ISurfaceDataService service;
@PostConstruct
public void init() {
@ -94,4 +95,12 @@ public class SurfaceWeatherController {
return Result.success(warning);
}
@PostMapping("/queryAllWeatherWarning")
@ControllerLog(info = "全国地面气象预警信息查询")
public Result queryWeatherWarning() throws Exception {
// List<SurfaceWeatherWarning> warning = surfaceDataService.getSurfaceWarning(lat, lon);
List<CmaWarning> cmaWarningList = cmaService.getSurfaceWeatherWarnings(null);
cmaService.updateCmaWarningCache(cmaWarningList);
return Result.success(cmaWarningList);
}
}

@ -4,10 +4,11 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.config.aspect.ControllerLog;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.*;
import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import com.htfp.weather.web.param.request.*;
import com.htfp.weather.web.param.response.*;
import com.htfp.weather.web.service.GfsUpperDataServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -24,11 +25,11 @@ import java.util.List;
@CrossOrigin
@RestController
@Slf4j
@RequestMapping("/htfp/weather/upper/")
@RequestMapping("/htfp/weather/v1/upper/")
public class UpperWeatherController {
@Resource(name = "tablestore-gfs")
GfsDataServiceImpl gfsDataService;
GfsUpperDataServiceImpl gfsDataService;
@PostMapping("/queryUpperNowWeather")
@ControllerLog(info = "高空实时气象信息查询")
@ -43,20 +44,13 @@ public class UpperWeatherController {
@PostMapping("/queryUpperNowWeatherInMultiPoints")
@ControllerLog(info = "高空航点实时气象信息查询")
public Result queryUpperNowWeatherInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
Double latitude = position.getLatitude();
Double longitude = position.getLongitude();
if (latitude != null && longitude != null) {
latitudeList.add(latitude);
longitudeList.add(longitude);
} else {
throw new AppException(ErrorCode.MULTI_INDEX_ERROR);
}
}
int level = multiPointsRequest.getLevel().intValue();
List<NowWeatherStatus> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(level, latitudeList, longitudeList);
List<Position2D> pointList = multiPointsRequest.getPointList();
int size = pointList.size();
List<Double> latitudeList = new ArrayList<>(size);
List<Double> longitudeList = new ArrayList<>(size);
checkLatitudeLongitude(latitudeList, longitudeList, pointList);
int level = multiPointsRequest.getLevel();
List<NowWeatherStatus> nowWeatherByMultiPoint = gfsDataService.getNowWeatherByMultiPoint(latitudeList, longitudeList, level);
return Result.success(nowWeatherByMultiPoint);
}
@ -73,19 +67,12 @@ public class UpperWeatherController {
@PostMapping("/queryUpperForecastInMultiPoints")
@ControllerLog(info = "高空航点24小时预报结果查询")
public Result queryUpperForecastInMultiPoints(@Validated @RequestBody MultiPointsRequest multiPointsRequest) {
List<Double> latitudeList = new ArrayList<>();
List<Double> longitudeList = new ArrayList<>();
for (Position2D position : multiPointsRequest.getPointList()) {
Double latitude = position.getLatitude();
Double longitude = position.getLongitude();
if (latitude != null && longitude != null) {
latitudeList.add(latitude);
longitudeList.add(longitude);
} else {
throw new AppException(ErrorCode.MULTI_INDEX_ERROR);
}
}
int level = multiPointsRequest.getLevel().intValue();
List<Position2D> pointList = multiPointsRequest.getPointList();
int size = pointList.size();
List<Double> latitudeList = new ArrayList<>(size);
List<Double> longitudeList = new ArrayList<>(size);
checkLatitudeLongitude(latitudeList, longitudeList, pointList);
int level = multiPointsRequest.getLevel();
List<TimeSeriesDataset> nowWeatherByMultiPoint = gfsDataService.getForecastSeriesByMultiPoint(latitudeList, longitudeList, level);
return Result.success(nowWeatherByMultiPoint);
}
@ -171,4 +158,22 @@ public class UpperWeatherController {
return Result.success(forecastSeries);
}
private void checkLatitudeLongitude(List<Double> latitudeList, List<Double> longitudeList, List<Position2D> pointList) {
if (CollectionUtils.isEmpty(pointList)) {
throw new AppException(ErrorCode.EMPTY_INDEX_ERROR, "航点列表为空");
}
int size = pointList.size();
for (Position2D position : pointList) {
Double latitude = position.getLatitude();
Double longitude = position.getLongitude();
if (latitude != null && longitude != null) {
latitudeList.add(latitude);
longitudeList.add(longitude);
} else {
throw new AppException(ErrorCode.MULTI_INDEX_ERROR, "经纬度不允许为空");
}
}
}
}

@ -1,5 +1,7 @@
package com.htfp.weather.web.exception;
import org.checkerframework.checker.units.qual.A;
/**
* @Author : shiyi
* @Date : 2024/3/14 16:30
@ -8,7 +10,7 @@ package com.htfp.weather.web.exception;
public enum ErrorCode {
//
OTHER_ERROR(-1, "其他错误"),
API_VALIDATE_ERROR(-1, "接口校验错误"),
VALIDATE_ERROR(1001, "参数校验错误 "),
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
@ -39,6 +41,7 @@ public enum ErrorCode {
NO_SUCH_VARIABLE(6006, "找不到变量名映射"),
TABLESTORE_QUERY_ERROR(6007, "数据库查询错误"),
RESPONSE_DATA_BUILD_ERROR(6008, "查询数据结果构建错误"),
EMPTY_INDEX_ERROR(6009, "坐标索引为空"),
;
private final int code;
private final String msg;

@ -1,9 +1,7 @@
package com.htfp.weather.web.pojo.caiyun;
package com.htfp.weather.web.param.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;

@ -1,9 +1,8 @@
package com.htfp.weather.web.pojo.caiyun;
package com.htfp.weather.web.param.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;

@ -1,9 +1,8 @@
package com.htfp.weather.web.pojo.caiyun;
package com.htfp.weather.web.param.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @Author : shiyi

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.caiyun;
package com.htfp.weather.web.param.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.cma;
package com.htfp.weather.web.param.cma;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.cma;
package com.htfp.weather.web.param.cma;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;

@ -1,9 +1,8 @@
package com.htfp.weather.web.pojo.hefeng;
package com.htfp.weather.web.param.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.jetbrains.annotations.NotNull;
import java.util.List;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.hefeng;
package com.htfp.weather.web.param.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.hefeng;
package com.htfp.weather.web.param.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import com.htfp.weather.download.gfs.GfsDataConfig;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.web.valid.EnumValid;
@ -14,8 +14,10 @@ import java.util.List;
*/
@Data
public class MultiPointsRequest {
@NotNull
List<Position2D> pointList;
@NotNull(message = "高度层次 (level) 不能为空")
@EnumValid(message = "高度层次 (level) 不存在", enumClass = GfsLevelsEnum.class)
@EnumValid(message = "高度层次 (level) 数值错误", enumClass = GfsLevelsEnum.class)
Integer level;
}

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.download.gfs.GfsLevelsEnum;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import lombok.Data;

@ -1,7 +1,5 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
@ -18,6 +16,6 @@ import lombok.EqualsAndHashCode;
@Data
public class Position3D extends Position2D {
@NotNull(message = "高度层次 (level) 不能为空")
@EnumValid(message = "高度层次 (level) 不存在", enumClass = GfsLevelsEnum.class)
@EnumValid(message = "高度层次 (level) 数值错误", enumClass = GfsLevelsEnum.class, usedField = "getCode")
Integer level;
}

@ -1,7 +1,6 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.info.Constant;
import com.htfp.weather.web.valid.DateTimeStr;
import com.htfp.weather.web.valid.EnumValid;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.request;
package com.htfp.weather.web.param.request;
import com.htfp.weather.griddata.common.TableConfig;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.htfp.weather.download.DownLoadFileInfo;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import lombok.Data;

@ -1,6 +1,5 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportFileInfo;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;

@ -1,11 +1,8 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;
import java.util.ArrayList;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:19

@ -1,8 +1,7 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;
import lombok.Getter;
/**
* @Author : shiyi

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.htfp.weather.web.exception.ErrorCode;
import lombok.Data;

@ -1,4 +1,4 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;

@ -1,9 +1,8 @@
package com.htfp.weather.web.pojo.response;
package com.htfp.weather.web.param.response;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Data;
import java.sql.Time;
import java.util.ArrayList;
import java.util.List;

@ -2,7 +2,7 @@ package com.htfp.weather.web.service;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.FileInfo;
import com.htfp.weather.web.param.response.FileInfo;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Component;

@ -18,10 +18,9 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.*;
import com.htfp.weather.web.param.response.*;
import com.htfp.weather.web.service.surfaceapi.ISurfaceDataService;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.units.qual.A;
import org.springframework.stereotype.Service;
import ucar.ma2.Array;
import ucar.ma2.Index4D;
@ -43,7 +42,7 @@ import java.util.stream.Collectors;
*/
@Service("tablestore-gfs")
@Slf4j
public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
public class GfsUpperDataServiceImpl implements IUpperDataService, ISurfaceDataService {
@Resource
GfsDataFetcher gfsDataFetcher;
@Resource
@ -100,7 +99,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
if (downloadResult.isDownloadSuccess()) {
return downloadResult;
} else {
throw new AppException(ErrorCode.DOWNLOAD_ERROR);
throw new AppException(ErrorCode.DOWNLOAD_ERROR, downloadResult.getErrorMsg());
}
}
}
@ -131,6 +130,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
* @param level 9999
* @return @{@link NowWeatherStatus}
*/
@Override
public NowWeatherStatus getNowWeather(int level, double latitude, double longitude) {
OffsetDateTime nowTime = OffsetDateTime.now();
ValueRange<OffsetDateTime> timeRange = new ValueRange<>(Collections.singletonList(nowTime));
@ -162,7 +162,8 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
* @param level
* @return
*/
public List<NowWeatherStatus> getNowWeatherByMultiPoint(int level, List<Double> latitude, List<Double> longitude) {
@Override
public List<NowWeatherStatus> getNowWeatherByMultiPoint(List<Double> latitude, List<Double> longitude, int level) {
if (latitude.size() != longitude.size()) {
throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致");
}
@ -202,6 +203,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
* @param level 9999
* @return @{@link TimeSeriesDataset}
*/
@Override
public TimeSeriesDataset getForecastSeries(int level, double latitude, double longitude) {
List<String> variableNames = getVariableNamesInDatabase(level);
OffsetDateTime nowTime = OffsetDateTime.now();
@ -230,6 +232,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
* @param level
* @return
*/
@Override
public List<TimeSeriesDataset> getForecastSeriesByMultiPoint(List<Double> latitudeList, List<Double> longitudeList, int level) {
if (latitudeList.size() != longitudeList.size()) {
throw new AppException(ErrorCode.INDEX_SIZE_ERROR, ": 经纬度数组大小不一致");
@ -291,6 +294,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
}
/** 获取全部变量随气压变化的廓线 (推荐) */
@Override
public ProfileDataset getProfileByPressure(OffsetDateTime targetTime, double latitude, double longitude) {
try {
GfsLevelsEnum levelFlag = GfsLevelsEnum.UPPER;
@ -362,6 +366,7 @@ public class GfsDataServiceImpl implements IDataService, ISurfaceDataService {
* @param maxLon
* @return @{@link PlaneResponse}
*/
@Override
public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) {
String targetVariable;
if (GfsLevelsEnum.SURFACE.equals(GfsLevelsEnum.getByCode(level))) {

@ -1,9 +0,0 @@
package com.htfp.weather.web.service;
/**
* @Author : shiyi
* @Date : 2024/3/15 10:51
* @Description :
*/
public interface IDataService {
}

@ -0,0 +1,28 @@
package com.htfp.weather.web.service;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.PlaneResponse;
import com.htfp.weather.web.param.response.ProfileDataset;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import java.time.OffsetDateTime;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/3/15 10:51
* @Description :
*/
public interface IUpperDataService {
NowWeatherStatus getNowWeather(int level, double latitude, double longitude);
List<NowWeatherStatus> getNowWeatherByMultiPoint(List<Double> latitude, List<Double> longitude, int level);
TimeSeriesDataset getForecastSeries(int level, double latitude, double longitude);
List<TimeSeriesDataset> getForecastSeriesByMultiPoint(List<Double> latitudeList, List<Double> longitudeList, int level);
ProfileDataset getProfileByPressure(OffsetDateTime targetTime, double latitude, double longitude);
PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon);
}

@ -5,13 +5,13 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.caiyun.*;
import com.htfp.weather.web.pojo.caiyun.CaiYunNowResponse.NowResult.RealTime;
import com.htfp.weather.web.pojo.caiyun.CaiYunForecastResponse.HourlyResult.Hourly;
import com.htfp.weather.web.pojo.caiyun.CaiYunWarningResponse.AlertResult.Alert.AlertContent;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.param.caiyun.*;
import com.htfp.weather.web.param.caiyun.CaiYunNowResponse.NowResult.RealTime;
import com.htfp.weather.web.param.caiyun.CaiYunForecastResponse.HourlyResult.Hourly;
import com.htfp.weather.web.param.caiyun.CaiYunWarningResponse.AlertResult.Alert.AlertContent;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.SurfaceWeatherWarning;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

@ -5,12 +5,12 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.JSONUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.cma.AntiGeoCodeResponse;
import com.htfp.weather.web.pojo.cma.AntiGeoCodeResponse.Result.AddressComponent;
import com.htfp.weather.web.pojo.cma.CmaWarning;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.param.cma.AntiGeoCodeResponse;
import com.htfp.weather.web.param.cma.AntiGeoCodeResponse.Result.AddressComponent;
import com.htfp.weather.web.param.cma.CmaWarning;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.SurfaceWeatherWarning;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -26,6 +26,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static com.htfp.weather.utils.HttpClientUtils.sendGet;
@ -42,6 +45,7 @@ public class CmaServiceImpl implements ISurfaceDataService{
@Value("${tianditu.key}") @Setter
private String keyOfTianDiTu;
private Map<String, List<SurfaceWeatherWarning>> warningCache = new ConcurrentHashMap<>();
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
@Override
public String getDataSource() {
return dataSource;
@ -103,12 +107,21 @@ public class CmaServiceImpl implements ISurfaceDataService{
@Scheduled(cron = "0 20,50 * * * ?")
@PostConstruct
public void updateCmaWarning() {
updateCmaWarning(null); // 更新全国预警数据
List<CmaWarning> cmaWarningList = null;// 更新全国预警数据
try {
cmaWarningList = getSurfaceWeatherWarnings(null);
} catch (Exception e) {
// 3分钟后重新执行此函数
scheduler.schedule(this::updateCmaWarning, 5, TimeUnit.MINUTES);
log.warn("获取全国预警数据失败3分钟后重新更新");
return;
}
updateCmaWarningCache(cmaWarningList);
}
/**
* @param countyCode */
public void updateCmaWarning(String countyCode) {
public List<CmaWarning> getSurfaceWeatherWarnings(String countyCode) {
String url = "https://data.cma.cn/dataGis/disasterWarning/getWarningDataByCnty";
// 6小时前12小时后
OffsetDateTime startTime = DateTimeUtils.offsetDateTimeToSystemZone(OffsetDateTime.now()).minusHours(6);
@ -118,13 +131,24 @@ public class CmaServiceImpl implements ISurfaceDataService{
params.put("startTime", startTime.format(formatter));
params.put("endTime", endTime.format(formatter));
params.put("provinceCode", countyCode);
log.info("更新预警数据: {}", params);
log.info("获取预警数据: {}", params);
try {
if (params.get("provinceCode") == null) {
warningCache.clear();
}
String jsonStr = sendGet(url, params);
List<CmaWarning> cmaWarningList = JSONUtils.json2list(jsonStr, CmaWarning.class);
return cmaWarningList;
} catch (Exception e) {
log.error("[中央气象台] 预警信息请求结果处理错误, 预警数据更新失败, {}", e.getMessage());
throw new AppException(ErrorCode.CMA_REQUEST_ERROR, e.getMessage());
}
}
/**
*
* @param cmaWarningList
*/
public void updateCmaWarningCache(List<CmaWarning> cmaWarningList) {
log.info("更新预警数据, 区域数量: {}", cmaWarningList.size());
warningCache.clear();
for (CmaWarning cmaWarning : cmaWarningList) {
String countyId = cmaWarning.getAreaId();
String dataid = cmaWarning.getDataid();
@ -138,11 +162,8 @@ public class CmaServiceImpl implements ISurfaceDataService{
warningCache.get(countyId).add(surfaceWeatherWarning);
}
}
} catch (Exception e) {
log.error("[中央气象台] 预警信息请求结果处理错误, 预警数据更新失败, {}", e.getMessage());
}
}
}
private SurfaceWeatherWarning buildSurfaceWeatherWarning(CmaWarning cmaWarning) {
SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning();
surfaceWeatherWarning.setSource(cmaWarning.getSender());

@ -5,13 +5,13 @@ import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.hefeng.*;
import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse.HeFengNow;
import com.htfp.weather.web.pojo.hefeng.HeFengForecastResponse.HeFengForecastHour;
import com.htfp.weather.web.pojo.hefeng.HeFengWarningResponse.HeFengWarning;
import com.htfp.weather.web.param.response.SurfaceWeatherWarning;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.hefeng.*;
import com.htfp.weather.web.param.hefeng.HeFengNowResponse.HeFengNow;
import com.htfp.weather.web.param.hefeng.HeFengForecastResponse.HeFengForecastHour;
import com.htfp.weather.web.param.hefeng.HeFengWarningResponse.HeFengWarning;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

@ -1,15 +1,15 @@
package com.htfp.weather.web.service.surfaceapi;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.SurfaceWeatherWarning;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/4/30 10:02
* @Description :
* @Description :
*/
public interface ISurfaceDataService {
String getDataSource();

@ -1,5 +1,5 @@
server:
port: 10324
port: 10323
spring:
profiles:
include: weather
@ -13,10 +13,14 @@ spring:
host: smtp.exmail.qq.com
username: shiyi@htsdfp.com
password: weatherHTFP123
port: 465 # 阿里云ECS为管控垃圾邮件屏蔽了默认的25端口号的服务
protocol: smtps
default-encoding: utf-8
properties.mail.smtp:
auth: true
enable: true
required: true
task:
scheduling:
pool:

@ -24,6 +24,5 @@ class GridDataProcessorTest {
@Test
void download() {
gridDataProcessor.download();
}
}

@ -1,14 +1,12 @@
package com.htfp.weather.utils;
import com.htfp.weather.web.pojo.hefeng.HeFengForecastResponse;
import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse;
import com.htfp.weather.web.param.hefeng.HeFengForecastResponse;
import com.htfp.weather.web.param.hefeng.HeFengNowResponse;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import static org.junit.jupiter.api.Assertions.assertThrows;
@ -77,7 +75,7 @@ class HttpClientUtilsTest {
params.put("location", location);
params.put("key", key);
String jsonStr = HttpClientUtils.sendGet(url, params);
System.out.println(JSONUtils.json2pojo(jsonStr, HeFengNowResponse.class));
System.out.println(JSONUtils.json2obj(jsonStr, HeFengNowResponse.class));
// assertThrows(IOException.class, ()->{HttpClientUtils.sendGet(url, params);});
}

@ -1,13 +1,11 @@
package com.htfp.weather.web.service;
import com.htfp.weather.web.pojo.response.FileInfo;
import com.htfp.weather.web.param.response.FileInfo;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/6/12 11:52

@ -1,8 +1,8 @@
package com.htfp.weather.web.service.surfaceapi;
import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.param.hefeng.HeFengNowResponse;
import com.htfp.weather.web.param.response.NowWeatherStatus;
import com.htfp.weather.web.param.response.TimeSeriesDataset;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Value;

Loading…
Cancel
Save