ShiYi
/
WeatherServer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first commit

Browse Source
download
shiyi 1 year ago
commit bc43f9f1ca
146 changed files with 8401 additions and 0 deletions
Show Stats Download Patch File Download Diff File

23
.gitignore vendored
Unescape Escape View File

@ -0,0 +1,23 @@
/GFSData/
/tablestoreConf.json
/log/
/*/target/
/tablestore-grid-master/tableConf.json
/tablestore-grid-master/tablestoreConf.json
/tablestore-grid-master/out/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr

122
tablestore-grid-master/pom.xml
Unescape Escape View File

@ -0,0 +1,122 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>unidata-all</id>
<name>Unidata All</name>
<url>https://artifacts.unidata.ucar.edu/repository/unidata-all/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>edu.ucar</groupId>
<artifactId>cdm-core</artifactId>
<version>5.5.3</version>
</dependency>
<dependency>
<groupId>edu.ucar</groupId>
<artifactId>grib</artifactId>
<version>5.5.3</version>
</dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>tablestore</artifactId>
<version>5.16.0</version>
<classifier>jar-with-dependencies</classifier>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.4.12</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.30</version>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to
parent pom) -->
<plugins>
<!-- clean lifecycle, see
https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see
https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<!-- site lifecycle, see
https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

54
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/CreateStoreExample.java
Unescape Escape View File

@ -0,0 +1,54 @@
package com.aliyun.tablestore.example.grid;
import com.alicloud.openservices.tablestore.model.search.FieldSchema;
import com.alicloud.openservices.tablestore.model.search.FieldType;
import com.alicloud.openservices.tablestore.model.search.IndexSchema;
import com.aliyun.tablestore.example.grid.common.ExampleConfig;
import com.aliyun.tablestore.example.grid.common.TableStoreGridExample;
import java.util.Arrays;
/**
* this example will create table and index.
*/
public class CreateStoreExample extends TableStoreGridExample {
public CreateStoreExample() {
super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME);
}
/**
* we must create store before we can use it.
* @throws Exception
*/
private void createStore() throws Exception {
this.tableStoreGrid.createStore();
}
/**
* this example create an index which contains these columns: status, tag1, tag2, create_time.
* you can create an index which contains any other columns.
*
* @throws Exception
*/
private void createIndex() throws Exception {
IndexSchema indexSchema = new IndexSchema();
indexSchema.setFieldSchemas(Arrays.asList(
new FieldSchema("status", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("tag1", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("tag2", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true)
));
this.tableStoreGrid.createMetaIndex(ExampleConfig.GRID_META_INDEX_NAME, indexSchema);
}
public static void main(String[] args) throws Exception {
CreateStoreExample example = new CreateStoreExample();
try {
example.createStore();
example.createIndex();
} finally {
example.close();
}
}
}

82
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataFetchExample.java
Unescape Escape View File

@ -0,0 +1,82 @@
package com.aliyun.tablestore.example.grid;
import com.aliyun.tablestore.example.grid.common.ExampleConfig;
import com.aliyun.tablestore.example.grid.common.TableStoreGridExample;
import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import ucar.ma2.Array;
import java.util.Arrays;
import java.util.Collections;
public class DataFetchExample extends TableStoreGridExample {
public DataFetchExample() {
super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME);
}
public Array queryByTableStore(String dataSetId, String variable, int[] origin, int[] shape) throws Exception {
GridDataFetcher fetcher = this.tableStoreGrid.getDataFetcher(this.tableStoreGrid.getDataSetMeta(dataSetId));
fetcher.setVariablesToGet(Collections.singletonList(variable));
fetcher.setOriginShape(origin, shape);
Grid4D grid4D = fetcher.fetch().getVariable(variable);
return grid4D.toArray();
}
/**
* get a plane.
* @throws Exception
*/
public void fetch1() throws Exception {
int[] origin = new int[] {0, 0, 0, 0};
int[] shape = new int[] {1, 1, ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[2], ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[3]};
Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID,
ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE,
origin, shape);
System.out.println("Shape: " + array.shapeToString());
System.out.println("Data: " + array);
}
/**
* get an sequence of point with different levels.
* @throws Exception
*/
public void fetch2() throws Exception {
int[] origin = new int[] {0, 0, 0, 0};
int[] shape = new int[] {1, ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[1], 1, 1};
Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID,
ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE,
origin, shape);
System.out.println("Shape:" + array.shapeToString());
System.out.println("Data:" + array);
}
/**
* get arbitrary 4-dimension data.
* @throws Exception
*/
public void fetch3() throws Exception {
int[] origin = new int[] {2, 5, 10, 10};
int[] shape = new int[] {3, 10, 30, 30};
Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID,
ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE,
origin, shape);
System.out.println("Shape:" + array.shapeToString());
System.out.println("Data:" + array);
}
public void run() throws Exception {
fetch1();
fetch2();
fetch3();
}
public static void main(String[] args) throws Exception {
DataFetchExample example = new DataFetchExample();
try {
example.run();
} finally {
example.close();
}
}
}

110
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataImportExample.java
Unescape Escape View File

@ -0,0 +1,110 @@
package com.aliyun.tablestore.example.grid;
import com.aliyun.tablestore.example.grid.common.ExampleConfig;
import com.aliyun.tablestore.example.grid.common.TableStoreGridExample;
import com.aliyun.tablestore.grid.GridDataWriter;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import lombok.extern.slf4j.Slf4j;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Slf4j
public class DataImportExample extends TableStoreGridExample {
public DataImportExample() {
super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME);
}
/**
* init meta data to table store.
* @param dataSetID
* @param dataType
* @param variables
* @param shape
* @return
* @throws Exception
*/
public GridDataSetMeta initMeta(String dataSetID, DataType dataType, List<String> variables, int[] shape) throws Exception {
GridDataSetMeta meta = new GridDataSetMeta(
dataSetID,
dataType,
variables,
shape[0],
shape[1],
shape[2],
shape[3],
new StoreOptions(StoreOptions.StoreType.SLICE));
meta.addAttribute("status", "INIT");
meta.addAttribute("create_time", System.currentTimeMillis());
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
/**
* update meta and set status to DONE when data import finished.
* @param meta
* @return
* @throws Exception
*/
public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception {
meta.addAttribute("status", "DONE");
tableStoreGrid.updateDataSetMeta(meta);
return meta;
}
/**
* read data from netcdf file and write data to table store.
* @param meta
* @param ncFileName
* @throws Exception
*/
public void importFromNcFile(GridDataSetMeta meta, String ncFileName) throws Exception {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
NetcdfFile ncFile = NetcdfFile.open(ncFileName);
List<Variable> variables = ncFile.getVariables();
for (Variable variable : variables) {
if (meta.getVariables().contains(variable.getShortName())) {
for (int t = 0; t < meta.gettSize(); t++) {
for (int z = 0; z < meta.getzSize(); z++) {
Array array = variable.read(new int[]{t, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[] {0, 0}, new int[] {meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variable.getShortName(), t, z, grid2D);
}
}
}
}
}
public void run() throws Exception {
String dataSetId = ExampleConfig.EXAMPLE_GRID_DATA_SET_ID;
String filePath = ExampleConfig.EXAMPLE_GRID_DATA_SET_NC_FILE_PATH;
String variable = ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE;
int[] shape = ExampleConfig.EXAMPLE_GRID_DATA_SHAPE;
DataType dataType = ExampleConfig.EXAMPLE_GRID_DATA_TYPE;
GridDataSetMeta meta = initMeta(dataSetId, dataType, Collections.singletonList(variable), shape);
importFromNcFile(meta, filePath);
updateMeta(meta);
}
public static void main(String[] args) throws Exception {
DataImportExample example = new DataImportExample();
try {
log.info("导入数据");
example.run();
} finally {
example.close();
}
}
}

59
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaDeleteExample.java
Unescape Escape View File

@ -0,0 +1,59 @@
package com.aliyun.tablestore.example.grid;
import com.alicloud.openservices.tablestore.model.search.sort.FieldSort;
import com.alicloud.openservices.tablestore.model.search.sort.Sort;
import com.alicloud.openservices.tablestore.model.search.sort.SortOrder;
import com.aliyun.tablestore.example.grid.common.ExampleConfig;
import com.aliyun.tablestore.example.grid.common.TableStoreGridExample;
import com.aliyun.tablestore.grid.core.QueryBuilder;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import lombok.extern.slf4j.Slf4j;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Slf4j
public class MetaDeleteExample extends TableStoreGridExample {
public MetaDeleteExample() {
super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME);
}
private List<GridDataSetMeta> query() throws Exception {
QueryGridDataSetResult result = tableStoreGrid.queryDataSets(
ExampleConfig.GRID_META_INDEX_NAME,
QueryBuilder.or()
.equal("status", "DONE")
.equal("status", "INIT")
.build(),
new QueryParams(0, 1000, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC)))));
return result.getGridDataSetMetas();
}
private void batchDeleteRow() throws Exception {
List<GridDataSetMeta> metaList = query();
while (!metaList.isEmpty()) {
for (GridDataSetMeta meta : metaList) {
// 获取行的主键
String gridDataSetId = meta.getGridDataSetId();
if (!"test_echam_spectral_example".equals(gridDataSetId)) {
log.info("删除行：{}", gridDataSetId);
tableStoreGrid.deleteDataSetMeta(gridDataSetId);
}
}
metaList = query();
}
}
public static void main(String[] args) throws Exception {
MetaDeleteExample example = new MetaDeleteExample();
try {
example.batchDeleteRow();
} finally {
example.close();
}
}
}

116
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaQueryExample.java
Unescape Escape View File

@ -0,0 +1,116 @@
package com.aliyun.tablestore.example.grid;
import com.alicloud.openservices.tablestore.model.search.sort.FieldSort;
import com.alicloud.openservices.tablestore.model.search.sort.Sort;
import com.alicloud.openservices.tablestore.model.search.sort.SortOrder;
import com.aliyun.tablestore.example.grid.common.ExampleConfig;
import com.aliyun.tablestore.example.grid.common.TableStoreGridExample;
import com.aliyun.tablestore.grid.core.QueryBuilder;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import com.aliyun.tablestore.grid.model.StoreOptions;
import ucar.ma2.DataType;
import java.util.*;
public class MetaQueryExample extends TableStoreGridExample {
public MetaQueryExample() {
super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME);
}
private GridDataSetMeta initMeta(String uniqueKey, DataType dataType, List<String> variables, int[] shape, Map<String, Object> attributes) throws Exception {
GridDataSetMeta meta = new GridDataSetMeta(
uniqueKey,
dataType,
variables,
shape[0],
shape[1],
shape[2],
shape[3],
new StoreOptions(StoreOptions.StoreType.SLICE));
meta.setAttributes(attributes);
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
/**
* write some metas to table store for test.
* @throws Exception
*/
private void prepareMetas() throws Exception {
for (String tagA : Arrays.asList("A", "B", "C")) {
for (String tagB : Arrays.asList("X", "Y", "Z")) {
for (int i = 0; i < 10; i++) {
String dataSetId = UUID.randomUUID().toString();
Map<String, Object> objectMap = new HashMap<String, Object>();
objectMap.put("create_time", System.currentTimeMillis());
objectMap.put("tag1", tagA);
objectMap.put("tag2", tagB);
objectMap.put("status", ((i % 2) == 0)?"INIT":"DONE");
initMeta(dataSetId, DataType.FLOAT, Collections.singletonList("var"), new int[]{1, 1, 1, 1}, objectMap);
}
}
}
}
/**
* query by arbitrary combination conditions.
* @throws Exception
*/
private void query() throws Exception {
/**
* query condition : (status == DONE) and (create_time > System.currentTimeMillis - 86400000) and (tag1 == A or tag2 == X)
* sort by create_time, desc.
*/
QueryGridDataSetResult result = tableStoreGrid.queryDataSets(
ExampleConfig.GRID_META_INDEX_NAME,
QueryBuilder.and()
.equal("status", "DONE")
.greaterThan("create_time", System.currentTimeMillis() - 86400000)
// .equal("accuracy", "1km")
.query(QueryBuilder.or()
.equal("tag1", "A")
.equal("tag2", "X")
// .equal("source", "ECMWF")
// .equal("source", "NMC")
.build())
.build(),
new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC)))));
System.out.println("GetMetaCount: " + result.getGridDataSetMetas().size());
for (GridDataSetMeta meta : result.getGridDataSetMetas()) {
System.out.println("Meta: " + meta.getGridDataSetId());
System.out.println(meta.getAttributes());
}
}
private void query1() throws Exception {
/**
* query condition : (status == DONE) and (create_time > System.currentTimeMillis - 86400000) and (tag1 == A or tag2 == X)
* sort by create_time, desc.
*/
QueryGridDataSetResult result = tableStoreGrid.queryDataSets(
"gfs_meta_table_index",
QueryBuilder.and()
.equal("status", "DONE")
.greaterThanEqual("reference_time", 1684368800000L)
.build(),
new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC)))));
System.out.println("GetMetaCount: " + result.getGridDataSetMetas().size());
for (GridDataSetMeta meta : result.getGridDataSetMetas()) {
System.out.println("Meta: " + meta.getGridDataSetId());
System.out.println(meta.getAttributes());
}
}
public static void main(String[] args) throws Exception {
MetaQueryExample example = new MetaQueryExample();
try {
System.out.println("Query...");
example.query1();
} finally {
example.close();
}
}
}

24
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/ExampleConfig.java
Unescape Escape View File

@ -0,0 +1,24 @@
package com.aliyun.tablestore.example.grid.common;
import ucar.ma2.DataType;
public class ExampleConfig {
/**
* 表名和index名
*/
public static String GRID_DATA_TABLE_NAME = "GRID_DATA_TABLE_EXAMPLE1";
public static String GRID_META_TABLE_NAME = "GRID_META_TABLE_EXAMPLE1";
public static String GRID_META_INDEX_NAME = "GRID_META_INDEX";
/**
* 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型
*/
public static String EXAMPLE_GRID_DATA_SET_ID = "test_echam_spectral_example";
public static String EXAMPLE_GRID_DATA_SET_NC_FILE_PATH = "test_echam_spectral.nc";
public static String EXAMPLE_GRID_DATA_VARIABLE = "tpot";
public static int[] EXAMPLE_GRID_DATA_SHAPE = new int[]{8, 47, 96, 192};
public static DataType EXAMPLE_GRID_DATA_TYPE = DataType.FLOAT;
}

62
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreConf.java
Unescape Escape View File

@ -0,0 +1,62 @@
package com.aliyun.tablestore.example.grid.common;
import com.google.gson.Gson;
import org.apache.commons.io.IOUtils;
import java.io.FileInputStream;
import java.io.InputStream;
public class TableStoreConf {
private String endpoint;
private String accessId;
private String accessKey;
private String instanceName;
/**
* 从目标配置文件中读取 数据库instance相关信息
*/
public static TableStoreConf newInstance(String path) {
try {
InputStream f = new FileInputStream(path);
Gson gson = new Gson();
return gson.fromJson(IOUtils.toString(f), TableStoreConf.class);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getAccessId() {
return accessId;
}
public void setAccessId(String accessId) {
this.accessId = accessId;
}
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getInstanceName() {
return instanceName;
}
public void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
}

33
tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreGridExample.java
Unescape Escape View File

@ -0,0 +1,33 @@
package com.aliyun.tablestore.example.grid.common;
import com.aliyun.tablestore.grid.TableStoreGrid;
import com.aliyun.tablestore.grid.TableStoreGridConfig;
public abstract class TableStoreGridExample {
protected TableStoreGrid tableStoreGrid;
// private String pathSeperator = "/";
public TableStoreGridExample(String dataTableName, String metaTableName) {
// String os = System.getProperty("os.name");
// if (os.toLowerCase().startsWith("win")) {
// pathSeparator = "\\";
// }
String pathSeparator = System.getProperty("file.separator");
TableStoreConf conf = TableStoreConf.newInstance(System.getProperty("user.dir") + pathSeparator + "tablestoreConf.json");
TableStoreGridConfig config = new TableStoreGridConfig();
config.setTableStoreEndpoint(conf.getEndpoint());
config.setAccessId(conf.getAccessId());
config.setAccessKey(conf.getAccessKey());
config.setTableStoreInstance(conf.getInstanceName());
config.setDataTableName(dataTableName);
config.setMetaTableName(metaTableName);
tableStoreGrid = new TableStoreGrid(config);
}
public void close() {
if (tableStoreGrid != null) {
tableStoreGrid.close();
}
}
}

10
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataDeleter.java
Unescape Escape View File

@ -0,0 +1,10 @@
package com.aliyun.tablestore.grid;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import java.util.concurrent.ExecutionException;
public interface GridDataDeleter {
void delete(String dataSetId) throws ExecutionException, InterruptedException;
}

47
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataFetcher.java
Unescape Escape View File

@ -0,0 +1,47 @@
package com.aliyun.tablestore.grid;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.grid.Range;
import java.util.Collection;
public interface GridDataFetcher {
/**
* 设置要查询的变量。
* @param variables
* @return
*/
GridDataFetcher setVariablesToGet(Collection<String> variables);
/**
* 设置要读取的各维度起始点和大小。
* @param origin 各维度起始点。
* @param shape 各维度大小。
* @return
*/
GridDataFetcher setOriginShape(int[] origin, int[] shape);
/**
* 获取数据。
* @return
* @throws Exception
*/
GridDataSet fetch() throws Exception;
GridDataFetcher setT(int t);
GridDataFetcher setTRange(Range tRange);
GridDataFetcher setZ(int z);
GridDataFetcher setZRange(Range zRange);
GridDataFetcher setX(int x);
GridDataFetcher setXRange(Range xRange);
GridDataFetcher setY(int y);
GridDataFetcher setYRange(Range yRange);
}

15
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataWriter.java
Unescape Escape View File

@ -0,0 +1,15 @@
package com.aliyun.tablestore.grid;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
public interface GridDataWriter {
/**
* 写入一个二维平面。
* @param variable 变量名。
* @param t 时间维的值。
* @param z 高度维的值。
* @param grid2D 平面数据。
* @throws Exception
*/
void writeGrid2D(String variable, int t, int z, Grid2D grid2D) throws Exception;
}

79
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridStore.java
Unescape Escape View File

@ -0,0 +1,79 @@
package com.aliyun.tablestore.grid;
import com.alicloud.openservices.tablestore.model.TableOptions;
import com.alicloud.openservices.tablestore.model.search.IndexSchema;
import com.alicloud.openservices.tablestore.model.search.query.Query;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
public interface GridStore {
/**
* 创建相关的meta、data表，数据录入前调用。
* @throws Exception
*/
void createStore() throws Exception;
void createStore(TableOptions tableOptions) throws Exception;
/**
* 写入gridDataSet的meta信息。
* @param meta
* @throws Exception
*/
void putDataSetMeta(GridDataSetMeta meta) throws Exception;
/**
* 更新meta信息。
* @param meta
* @throws Exception
*/
void updateDataSetMeta(GridDataSetMeta meta) throws Exception;
/**
* 通过gridDataSetId获取meta。
* @param dataSetId
* @return
* @throws Exception
*/
GridDataSetMeta getDataSetMeta(String dataSetId) throws Exception;
/**
* 创建meta表的多元索引。
* @param indexName
* @param indexSchema
* @throws Exception
*/
void createMetaIndex(String indexName, IndexSchema indexSchema) throws Exception;
/**
* 通过多种查询条件来查询符合条件的数据集。
* @param indexName 多元索引名。
* @param query 查询条件，可以通过QueryBuilder构建。
* @param queryParams 查询相关参数，包括offset、limit、sort等。
* @return
* @throws Exception
*/
QueryGridDataSetResult queryDataSets(String indexName, Query query, QueryParams queryParams) throws Exception;
/**
* 获取GridDataWriter用于写入数据。
* @param meta
* @return
*/
GridDataWriter getDataWriter(GridDataSetMeta meta);
/**
* 获取GridDataFetcher用于读取数据。
* @param meta
* @return
*/
GridDataFetcher getDataFetcher(GridDataSetMeta meta);
/**
* 释放资源。
*/
void close();
}

191
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGrid.java
Unescape Escape View File

@ -0,0 +1,191 @@
package com.aliyun.tablestore.grid;
import com.alicloud.openservices.tablestore.*;
import com.alicloud.openservices.tablestore.core.ErrorCode;
import com.alicloud.openservices.tablestore.model.GetRowResponse;
import com.alicloud.openservices.tablestore.model.Row;
import com.alicloud.openservices.tablestore.model.TableOptions;
import com.alicloud.openservices.tablestore.model.UpdateTableRequest;
import com.alicloud.openservices.tablestore.model.search.CreateSearchIndexRequest;
import com.alicloud.openservices.tablestore.model.search.IndexSchema;
import com.alicloud.openservices.tablestore.model.search.SearchResponse;
import com.alicloud.openservices.tablestore.model.search.query.Query;
import com.alicloud.openservices.tablestore.writer.WriterConfig;
import com.aliyun.tablestore.grid.core.TableStoreDataDeleter;
import com.aliyun.tablestore.grid.core.RequestBuilder;
import com.aliyun.tablestore.grid.core.RowParser;
import com.aliyun.tablestore.grid.core.TableStoreDataFetcher;
import com.aliyun.tablestore.grid.core.TableStoreDataWriter;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import ucar.nc2.ft.point.standard.TableConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class TableStoreGrid implements GridStore {
private final TableStoreGridConfig config;
private final AsyncClientInterface asyncClient;
private ExecutorService writerExecutor;
private TableStoreWriter writer;
private TableOptions tableOptions;
public TableStoreGrid(TableStoreGridConfig config) {
this.config = config;
this.asyncClient = new AsyncClient(config.getTableStoreEndpoint(),
config.getAccessId(), config.getAccessKey(),
config.getTableStoreInstance());
}
@Override
public void createStore() throws Exception {
this.tableOptions = new TableOptions(-1, 1);
createStore(tableOptions);
}
@Override
public void createStore(TableOptions tableOptions) throws Exception {
// create meta table
try {
this.tableOptions = tableOptions;
this.asyncClient.createTable(RequestBuilder.buildCreateMetaTableRequest(config.getMetaTableName(), tableOptions), null).get();
} catch (TableStoreException ex) {
if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) {
throw ex;
}
}
// create buffer table
try {
tableOptions.setAllowUpdate(false); // 禁止update操作
this.asyncClient.createTable(RequestBuilder.buildCreateDataTableRequest(config.getDataTableName(), tableOptions), null).get();
} catch (TableStoreException ex) {
if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) {
throw ex;
}
}
}
/**
* 更新表设置
* @param tableOptionsForUpdate
* @throws Exception
*/
public void updateStoreOption(TableOptions tableOptionsForUpdate) throws Exception {
// create meta table
try {
UpdateTableRequest metaUpdate = new UpdateTableRequest(config.getMetaTableName());
metaUpdate.setTableOptionsForUpdate(tableOptionsForUpdate);
this.asyncClient.updateTable(metaUpdate, null).get();
} catch (TableStoreException ex) {
if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) {
throw ex;
}
}
// create buffer table
try {
UpdateTableRequest dataUpdate = new UpdateTableRequest(config.getDataTableName());
dataUpdate.setTableOptionsForUpdate(tableOptionsForUpdate);
this.asyncClient.updateTable(dataUpdate, null).get();
} catch (TableStoreException ex) {
if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) {
throw ex;
}
}
}
@Override
public void putDataSetMeta(GridDataSetMeta meta) throws Exception {
this.asyncClient.putRow(
RequestBuilder.buildPutMetaRequest(config.getMetaTableName(), meta), null).get();
}
public void deleteDataSetMeta(String uniqueKey) throws Exception {
this.asyncClient.deleteRow(
RequestBuilder.buildDeleteMetaRequest(config.getMetaTableName(), uniqueKey),
null
);
}
@Override
public void updateDataSetMeta(GridDataSetMeta meta) throws Exception {
this.asyncClient.updateRow(
RequestBuilder.buildUpdateMetaRequest(config.getMetaTableName(), meta), null).get();
}
@Override
public GridDataSetMeta getDataSetMeta(String gridDataSetId) throws Exception {
GetRowResponse getRowResponse = this.asyncClient.getRow(
RequestBuilder.buildGetMetaRequest(config.getMetaTableName(), gridDataSetId), null).get();
if (getRowResponse.getRow() == null) {
return null;
}
return RowParser.parseMetaFromRow(getRowResponse.getRow());
}
@Override
public void createMetaIndex(String indexName, IndexSchema indexSchema) throws Exception {
CreateSearchIndexRequest request = new CreateSearchIndexRequest();
request.setTableName(config.getMetaTableName());
request.setIndexName(indexName);
request.setIndexSchema(indexSchema);
request.setTimeToLive(tableOptions.getTimeToLive(), TimeUnit.SECONDS);
this.asyncClient.createSearchIndex(request, null).get();
}
@Override
public QueryGridDataSetResult queryDataSets(String indexName, Query query, QueryParams queryParams) throws Exception {
SearchResponse searchResponse = this.asyncClient.search(
RequestBuilder.buildSearchRequest(config.getMetaTableName(), indexName, query, queryParams),
null).get();
List<GridDataSetMeta> metaList = new ArrayList<GridDataSetMeta>();
for (Row row : searchResponse.getRows()) {
metaList.add(RowParser.parseMetaFromRow(row));
}
QueryGridDataSetResult result = new QueryGridDataSetResult();
result.setGridDataSetMetas(metaList);
result.setAllSuccess(searchResponse.isAllSuccess());
result.setNextToken(searchResponse.getNextToken());
result.setTotalCount(searchResponse.getTotalCount());
return result;
}
@Override
public GridDataWriter getDataWriter(GridDataSetMeta meta) {
if (writer == null) {
synchronized (this) {
if (writer == null) {
this.writerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
this.writer = new DefaultTableStoreWriter(this.asyncClient, config.getDataTableName(), new WriterConfig(), null, this.writerExecutor);
}
}
}
return new TableStoreDataWriter(writer, config.getDataTableName(), meta);
}
@Override
public GridDataFetcher getDataFetcher(GridDataSetMeta meta) {
return new TableStoreDataFetcher(asyncClient, config.getDataTableName(), meta, config.getDataSizeLimitForFetch());
}
public GridDataDeleter getDataDeleter(GridDataSetMeta meta) {
return new TableStoreDataDeleter(asyncClient, config.getDataTableName(), meta);
}
@Override
public synchronized void close() {
if (writer != null) {
this.writer.close();
this.writerExecutor.shutdown();
}
this.asyncClient.shutdown();
}
}

70
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGridConfig.java
Unescape Escape View File

@ -0,0 +1,70 @@
package com.aliyun.tablestore.grid;
public class TableStoreGridConfig {
private String tableStoreEndpoint;
private String accessId;
private String accessKey;
private String tableStoreInstance;
private String metaTableName;
private String dataTableName;
private long dataSizeLimitForFetch = 20 * 1024 * 1024;
public String getTableStoreEndpoint() {
return tableStoreEndpoint;
}
public void setTableStoreEndpoint(String tableStoreEndpoint) {
this.tableStoreEndpoint = tableStoreEndpoint;
}
public String getAccessId() {
return accessId;
}
public void setAccessId(String accessId) {
this.accessId = accessId;
}
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getTableStoreInstance() {
return tableStoreInstance;
}
public void setTableStoreInstance(String tableStoreInstance) {
this.tableStoreInstance = tableStoreInstance;
}
public String getMetaTableName() {
return metaTableName;
}
public void setMetaTableName(String metaTableName) {
this.metaTableName = metaTableName;
}
public String getDataTableName() {
return dataTableName;
}
public void setDataTableName(String dataTableName) {
this.dataTableName = dataTableName;
}
public long getDataSizeLimitForFetch() {
return dataSizeLimitForFetch;
}
public void setDataSizeLimitForFetch(long dataSizeLimitForFetch) {
this.dataSizeLimitForFetch = dataSizeLimitForFetch;
}
}

29
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/AttributionEnum.java
Unescape Escape View File

@ -0,0 +1,29 @@
package com.aliyun.tablestore.grid.consts;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:27
* @Description : meta表属性枚举
*/
public enum AttributionEnum {
//
STATUS("status", "导入状态"),
CREATE_TIME("create_time", "创建时间"),
REFERENCE_TIME("reference_time", "起报时间"),
;
public String name;
public String info;
public String getName() {
return name;
}
public String getInfo() {
return info;
}
AttributionEnum(String name, String info) {
this.name = name;
this.info = info;
}
}

26
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/Constants.java
Unescape Escape View File

@ -0,0 +1,26 @@
package com.aliyun.tablestore.grid.consts;
public class Constants {
public static String GRID_DATA_SET_ID_PK_NAME = "_id";
public static String VARIABLE_PK_NAME = "_variable";
public static String T_PK_NAME = "_t";
public static String Z_PK_NAME = "_z";
public static String DATA_TYPE_COL_NAME = "_data_type";
public static String VARIABLE_LIST_COL_NAME = "_variable_list";
public static String T_SIZE_COL_NAME = "_t_size";
public static String Z_SIZE_COL_NAME = "_z_size";
public static String X_SIZE_COL_NAME = "_x_size";
public static String Y_SIZE_COL_NAME = "_y_size";
public static String STORE_TYPE_COL_NAME = "_store_type";
public static String X_SPLIT_COUNT_COL_NAME = "_x_split_count";
public static String Y_SPLIT_COUNT_COL_NAME = "_y_split_count";
public static String DATA_BLOCK_COL_NAME_PREFIX = "block_";
public static String DATA_BLOCK_COL_NAME_FORMAT = "block_%s_%s";
public static String T_LIST_COL_NAME = "time_list"; // 1M
public static int MAX_REQUEST_SIZE = 2 * 1024 * 1024;
}

134
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/QueryBuilder.java
Unescape Escape View File

@ -0,0 +1,134 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.model.ColumnValue;
import com.alicloud.openservices.tablestore.model.search.query.*;
import com.aliyun.tablestore.grid.utils.ValueUtil;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class QueryBuilder {
enum Operator {
AND,
OR
}
private final Operator operator;
private List<Query> filterQueries;
private List<Query> shouldQueries;
QueryBuilder(Operator operator) {
this.operator = operator;
switch (operator) {
case AND: {
this.filterQueries = new ArrayList<Query>();
break;
}
case OR: {
this.shouldQueries = new ArrayList<Query>();
break;
}
default:
throw new IllegalArgumentException();
}
}
public static QueryBuilder or() {
return new QueryBuilder(Operator.OR);
}
public static QueryBuilder and() {
return new QueryBuilder(Operator.AND);
}
public QueryBuilder equal(String columnName, Object... values) {
TermsQuery termsQuery = new TermsQuery();
termsQuery.setFieldName(columnName);
List<ColumnValue> columnValues = new ArrayList<ColumnValue>();
for (Object value : values) {
columnValues.add(ValueUtil.toColumnValue(value));
}
termsQuery.setTerms(columnValues);
return query(termsQuery);
}
public QueryBuilder notEqual(String columnName, Object value) {
TermQuery termQuery = new TermQuery();
termQuery.setFieldName(columnName);
termQuery.setTerm(ValueUtil.toColumnValue(value));
BoolQuery boolQuery = new BoolQuery();
boolQuery.setMustNotQueries(Collections.singletonList(termQuery));
return query(boolQuery);
}
public QueryBuilder greaterThan(String columnName, Object value) {
RangeQuery rangeQuery = new RangeQuery();
rangeQuery.setFieldName(columnName);
rangeQuery.greaterThan(ValueUtil.toColumnValue(value));
return query(rangeQuery);
}
public QueryBuilder greaterThanEqual(String columnName, Object value) {
RangeQuery rangeQuery = new RangeQuery();
rangeQuery.setFieldName(columnName);
rangeQuery.greaterThanOrEqual(ValueUtil.toColumnValue(value));
return query(rangeQuery);
}
public QueryBuilder lessThan(String columnName, Object value) {
RangeQuery rangeQuery = new RangeQuery();
rangeQuery.setFieldName(columnName);
rangeQuery.lessThan(ValueUtil.toColumnValue(value));
return query(rangeQuery);
}
public QueryBuilder lessThanEqual(String columnName, Object value) {
RangeQuery rangeQuery = new RangeQuery();
rangeQuery.setFieldName(columnName);
rangeQuery.lessThanOrEqual(ValueUtil.toColumnValue(value));
return query(rangeQuery);
}
public QueryBuilder prefix(String columnName, String prefix) {
PrefixQuery prefixQuery = new PrefixQuery();
prefixQuery.setFieldName(columnName);
prefixQuery.setPrefix(prefix);
return query(prefixQuery);
}
public QueryBuilder query(Query query) {
switch (operator) {
case AND: {
this.filterQueries.add(query);
break;
}
case OR: {
this.shouldQueries.add(query);
break;
}
default:
throw new IllegalArgumentException();
}
return this;
}
public Query build() {
switch (operator) {
case AND: {
BoolQuery boolQuery = new BoolQuery();
boolQuery.setFilterQueries(filterQueries);
return boolQuery;
}
case OR: {
BoolQuery boolQuery = new BoolQuery();
boolQuery.setShouldQueries(shouldQueries);
return boolQuery;
}
default:
throw new IllegalArgumentException();
}
}
}

149
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RequestBuilder.java
Unescape Escape View File

@ -0,0 +1,149 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.core.utils.StringUtils;
import com.alicloud.openservices.tablestore.model.*;
import com.alicloud.openservices.tablestore.model.search.SearchQuery;
import com.alicloud.openservices.tablestore.model.search.SearchRequest;
import com.alicloud.openservices.tablestore.model.search.query.Query;
import com.aliyun.tablestore.grid.model.*;
import com.aliyun.tablestore.grid.utils.ValueUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static com.aliyun.tablestore.grid.consts.Constants.*;
public class RequestBuilder {
public static CreateTableRequest buildCreateMetaTableRequest(String tableName, TableOptions tableOptions) {
TableMeta meta = new TableMeta(tableName);
meta.addPrimaryKeyColumn(new PrimaryKeySchema(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyType.STRING));
return new CreateTableRequest(meta, tableOptions);
}
public static CreateTableRequest buildCreateDataTableRequest(String tableName, TableOptions tableOptions) {
TableMeta meta = new TableMeta(tableName);
meta.addPrimaryKeyColumn(new PrimaryKeySchema(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyType.STRING));
meta.addPrimaryKeyColumn(new PrimaryKeySchema(VARIABLE_PK_NAME, PrimaryKeyType.STRING));
meta.addPrimaryKeyColumn(new PrimaryKeySchema(T_PK_NAME, PrimaryKeyType.INTEGER));
meta.addPrimaryKeyColumn(new PrimaryKeySchema(Z_PK_NAME, PrimaryKeyType.INTEGER));
return new CreateTableRequest(meta, tableOptions);
}
private static List<Column> buildMetaColumns(GridDataSetMeta meta) {
List<Column> columns = new ArrayList<Column>();
columns.add(new Column(DATA_TYPE_COL_NAME, ColumnValue.fromString(meta.getDataType().toString())));
columns.add(new Column(VARIABLE_LIST_COL_NAME, ColumnValue.fromString(StringUtils.join(",", meta.getVariables()))));
columns.add(new Column(T_SIZE_COL_NAME, ColumnValue.fromLong(meta.gettSize())));
columns.add(new Column(Z_SIZE_COL_NAME, ColumnValue.fromLong(meta.getzSize())));
columns.add(new Column(X_SIZE_COL_NAME, ColumnValue.fromLong(meta.getxSize())));
columns.add(new Column(Y_SIZE_COL_NAME, ColumnValue.fromLong(meta.getySize())));
columns.add(new Column(STORE_TYPE_COL_NAME, ColumnValue.fromString(meta.getStoreOptions().getStoreType().name())));
if (meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) {
columns.add(new Column(X_SPLIT_COUNT_COL_NAME, ColumnValue.fromLong(meta.getStoreOptions().getxSplitCount())));
columns.add(new Column(Y_SPLIT_COUNT_COL_NAME, ColumnValue.fromLong(meta.getStoreOptions().getySplitCount())));
}
if (meta.getAttributes() != null) {
for (Map.Entry<String, Object> entry : meta.getAttributes().entrySet()) {
columns.add(new Column(entry.getKey(), ValueUtil.toColumnValue(entry.getValue())));
}
}
if (meta.getForecastHours() != null) {
columns.add(new Column(T_LIST_COL_NAME, ColumnValue.fromString(StringUtils.join(",", meta.getForecastHours()))));
}
return columns;
}
public static PutRowRequest buildPutMetaRequest(String metaTableName, GridDataSetMeta meta) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId()));
PrimaryKey pk = builder.build();
RowPutChange rowPutChange = new RowPutChange(metaTableName, pk);
rowPutChange.addColumns(buildMetaColumns(meta));
return new PutRowRequest(rowPutChange);
}
public static UpdateRowRequest buildUpdateMetaRequest(String metaTableName, GridDataSetMeta meta) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId()));
PrimaryKey pk = builder.build();
RowUpdateChange rowUpdateChange = new RowUpdateChange(metaTableName, pk);
rowUpdateChange.put(buildMetaColumns(meta));
return new UpdateRowRequest(rowUpdateChange);
}
public static GetRowRequest buildGetMetaRequest(String metaTableName, String uniqueKey) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(uniqueKey));
PrimaryKey pk = builder.build();
SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(metaTableName);
criteria.setMaxVersions(1);
criteria.setPrimaryKey(pk);
GetRowRequest getRowRequest = new GetRowRequest();
getRowRequest.setRowQueryCriteria(criteria);
return getRowRequest;
}
public static SearchRequest buildSearchRequest(String metaTableName, String indexName, Query query, QueryParams params) {
SearchQuery searchQuery = new SearchQuery();
searchQuery.setQuery(query);
if (params.getOffset() != null) {
searchQuery.setOffset(params.getOffset());
}
if (params.getLimit() != null) {
searchQuery.setLimit(params.getLimit());
}
if (params.getSort() != null) {
searchQuery.setSort(params.getSort());
}
if (params.getToken() != null) {
searchQuery.setToken(params.getToken());
}
searchQuery.setGetTotalCount(params.isGetTotalCount());
SearchRequest searchRequest = new SearchRequest(metaTableName, indexName, searchQuery);
SearchRequest.ColumnsToGet columnsToGet = new SearchRequest.ColumnsToGet();
columnsToGet.setReturnAll(true);
searchRequest.setColumnsToGet(columnsToGet);
return searchRequest;
}
public static GetRowRequest buildGetDataRequest(GetDataParam getDataParam) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(getDataParam.getDataSetId()));
builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(getDataParam.getVariable()));
builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(getDataParam.getT()));
builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(getDataParam.getZ()));
PrimaryKey pk = builder.build();
SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(getDataParam.getDataTableName());
criteria.setMaxVersions(1);
criteria.setPrimaryKey(pk);
if (getDataParam.getColumnsToGet() != null) {
criteria.addColumnsToGet(getDataParam.getColumnsToGet());
}
return new GetRowRequest(criteria);
}
public static DeleteRowRequest buildDeleteDataRequest(DeleteDataParam deleteDataParam) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(deleteDataParam.getDataSetId()));
builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(deleteDataParam.getVariable()));
builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(deleteDataParam.getT()));
builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(deleteDataParam.getZ()));
PrimaryKey pk = builder.build();
RowDeleteChange rowDeleteChange = new RowDeleteChange(deleteDataParam.getDataTableName(), pk);
return new DeleteRowRequest(rowDeleteChange);
}
public static DeleteRowRequest buildDeleteMetaRequest(String metaTableName, String uniqueKey) {
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(uniqueKey));
PrimaryKey pk = builder.build();
SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(metaTableName);
criteria.setMaxVersions(1);
criteria.setPrimaryKey(pk);
RowDeleteChange rowDeleteChange = new RowDeleteChange(metaTableName, pk);
return new DeleteRowRequest(rowDeleteChange);
}
}

75
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RowParser.java
Unescape Escape View File

@ -0,0 +1,75 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.model.Column;
import com.alicloud.openservices.tablestore.model.Row;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.aliyun.tablestore.grid.model.grid.Plane;
import com.aliyun.tablestore.grid.model.grid.Range;
import com.aliyun.tablestore.grid.utils.BlockUtil;
import com.aliyun.tablestore.grid.utils.ValueUtil;
import ucar.ma2.DataType;
import java.nio.ByteBuffer;
import java.util.*;
import static com.aliyun.tablestore.grid.consts.Constants.*;
public class RowParser {
public static GridDataSetMeta parseMetaFromRow(Row row) {
String uniqueKey = row.getPrimaryKey().getPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME).getValue().asString();
DataType dataType = DataType.getType(row.getColumn(DATA_TYPE_COL_NAME).get(0).getValue().asString());
List<String> variables = Arrays.asList(row.getColumn(VARIABLE_LIST_COL_NAME).get(0).getValue().asString().split(","));
int tSize = (int) row.getColumn(T_SIZE_COL_NAME).get(0).getValue().asLong();
int zSize = (int) row.getColumn(Z_SIZE_COL_NAME).get(0).getValue().asLong();
int xSize = (int) row.getColumn(X_SIZE_COL_NAME).get(0).getValue().asLong();
int ySize = (int) row.getColumn(Y_SIZE_COL_NAME).get(0).getValue().asLong();
List<String> refTimes = Arrays.asList(row.getColumn(T_LIST_COL_NAME).get(0).getValue().asString().split(","));
StoreOptions.StoreType storeType = StoreOptions.StoreType.valueOf(
row.getColumn(STORE_TYPE_COL_NAME).get(0).getValue().asString());
StoreOptions storeOptions = new StoreOptions(storeType);
if (storeType.equals(StoreOptions.StoreType.SLICE)) {
storeOptions.setxSplitCount((int) row.getColumn(X_SPLIT_COUNT_COL_NAME).get(0).getValue().asLong());
storeOptions.setySplitCount((int) row.getColumn(Y_SPLIT_COUNT_COL_NAME).get(0).getValue().asLong());
}
Map<String, Object> attributes = new HashMap<String, Object>();
for (Column column : row.getColumns()) {
if (!column.getName().startsWith("_")) {
attributes.put(column.getName(), ValueUtil.toObject(column.getValue()));
}
}
GridDataSetMeta meta = new GridDataSetMeta(uniqueKey, dataType, variables, tSize, zSize, xSize, ySize, storeOptions);
meta.setForecastHours(refTimes);
meta.setAttributes(attributes);
return meta;
}
public static Grid2D parseGridFromRow(Row row, Plane plane, GridDataSetMeta meta, byte[] buffer, int pos) {
if (!meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) {
throw new IllegalArgumentException("unsupported store type");
}
int blockXSize = (meta.getxSize() - 1) / meta.getStoreOptions().getxSplitCount() + 1;
int blockYSize = (meta.getySize() - 1) / meta.getStoreOptions().getySplitCount() + 1;
List<Grid2D> blocks = new ArrayList<Grid2D>();
for (Column column : row.getColumns()) {
if (column.getName().startsWith(DATA_BLOCK_COL_NAME_PREFIX)) {
String[] strs = column.getName().split("_");
int xStart = Integer.valueOf(strs[strs.length - 2]);
int yStart = Integer.valueOf(strs[strs.length - 1]);
Range xRange = new Range(xStart, Math.min(xStart + blockXSize, meta.getxSize()));
Range yRange = new Range(yStart, Math.min(yStart + blockYSize, meta.getySize()));
int[] origin = new int[] {xStart, yStart};
int[] shape = new int[] {xRange.getSize(), yRange.getSize()};
Grid2D grid2D = new Grid2D(ByteBuffer.wrap(column.getValue().asBinary()), meta.getDataType(), origin, shape);
blocks.add(grid2D);
}
}
Grid2D grid2D = BlockUtil.buildGrid2DFromBlocks(plane, meta.getDataType(), blocks, buffer, pos);
return grid2D;
}
}

68
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataDeleter.java
Unescape Escape View File

@ -0,0 +1,68 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.AsyncClientInterface;
import com.alicloud.openservices.tablestore.model.BatchWriteRowRequest;
import com.alicloud.openservices.tablestore.model.BatchWriteRowResponse;
import com.aliyun.tablestore.grid.GridDataDeleter;
import com.aliyun.tablestore.grid.model.DeleteDataParam;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import lombok.extern.slf4j.Slf4j;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/**
* @Author : shiyi
* @Date : 2024/2/3 11:51
* @Description :
*/
@Slf4j
public class TableStoreDataDeleter implements GridDataDeleter {
private final AsyncClientInterface asyncClient;
private final GridDataSetMeta meta;
private final String dataTableName;
private final Collection<String> variables;
public TableStoreDataDeleter(AsyncClientInterface asyncClient, String dataTableName, GridDataSetMeta meta) {
this.asyncClient = asyncClient;
this.dataTableName = dataTableName;
this.meta = meta;
this.variables = this.meta.getVariables();
}
@Override
public void delete(String dataSetId) throws ExecutionException, InterruptedException {
Set<Integer> refTimes = meta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet());
for (String variable : meta.getVariables()) {
for (int t = 0; t < meta.gettSize(); t++) {
deleteByTime(dataSetId, variable, t);
}
}
}
private void deleteByTime(String dataSetId, String variable, int t) throws InterruptedException, ExecutionException {
BatchWriteRowRequest batchWriteRowRequest = new BatchWriteRowRequest();
for (int z = 0; z < meta.getzSize(); z++) {
DeleteDataParam param = new DeleteDataParam(dataTableName, dataSetId, variable, t, z);
// 添加到batch操作中, 如果该行不存在，也不会报错
batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange());
}
BatchWriteRowResponse response = asyncClient.batchWriteRow(batchWriteRowRequest, null).get();
log.info("variable={}, t={} 是否删除成功：{}", variable, t, response.isAllSucceed());
if (!response.isAllSucceed()) {
for (BatchWriteRowResponse.RowResult rowResult : response.getFailedRows()) {
log.warn("失败的行：{}", batchWriteRowRequest.getRowChange(rowResult.getTableName(), rowResult.getIndex()).getPrimaryKey());
log.warn("失败原因：{}", rowResult.getError());
}
/**
* 可以通过createRequestForRetry方法再构造一个请求对失败的行进行重试。此处只给出构造重试请求的部分。
* 推荐的重试方法是使用SDK的自定义重试策略功能，支持对batch操作的部分行错误进行重试。设置重试策略后，调用接口处无需增加重试代码。
*/
BatchWriteRowRequest retryRequest = batchWriteRowRequest.createRequestForRetry(response.getFailedRows());
}
}
}

224
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataFetcher.java
Unescape Escape View File

@ -0,0 +1,224 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.AsyncClientInterface;
import com.alicloud.openservices.tablestore.TableStoreCallback;
import com.alicloud.openservices.tablestore.model.GetRowRequest;
import com.alicloud.openservices.tablestore.model.GetRowResponse;
import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.model.GetDataParam;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.*;
import com.aliyun.tablestore.grid.utils.BlockUtil;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import static com.aliyun.tablestore.grid.consts.Constants.DATA_BLOCK_COL_NAME_FORMAT;
/**
* not thread-safe
*/
public class TableStoreDataFetcher implements GridDataFetcher {
private final AsyncClientInterface asyncClient;
private final GridDataSetMeta meta;
private final String tableName;
private final long dataSizeLimitForFetch;
private Collection<String> variables;
private Range tRange;
private Range zRange;
private Range xRange;
private Range yRange;
public TableStoreDataFetcher(AsyncClientInterface asyncClient, String tableName, GridDataSetMeta meta, long dataSizeLimitForFetch) {
this.asyncClient = asyncClient;
this.tableName = tableName;
this.meta = meta;
this.dataSizeLimitForFetch = dataSizeLimitForFetch;
this.variables = this.meta.getVariables();
this.tRange = new Range(0, this.meta.gettSize());
this.zRange = new Range(0, this.meta.getzSize());
this.xRange = new Range(0, this.meta.getxSize());
this.yRange = new Range(0, this.meta.getySize());
}
@Override
public GridDataFetcher setVariablesToGet(Collection<String> variables) {
this.variables = variables;
return this;
}
@Override
public GridDataFetcher setT(int t) {
return setTRange(new Range(t, t+1));
}
@Override
public GridDataFetcher setTRange(Range range) {
if (range.getStart() < 0 || range.getEnd() > meta.gettSize()) {
throw new IllegalArgumentException("range invalid");
}
this.tRange = range;
return this;
}
@Override
public GridDataFetcher setZ(int z) {
return setZRange(new Range(z, z+1));
}
@Override
public GridDataFetcher setZRange(Range range) {
if (range.getStart() < 0 || range.getEnd() > meta.getzSize()) {
throw new IllegalArgumentException("range invalid");
}
this.zRange = range;
return this;
}
@Override
public GridDataFetcher setX(int x) {
return setXRange(new Range(x, x+1));
}
@Override
public GridDataFetcher setXRange(Range range) {
if (range.getStart() < 0 || range.getEnd() > meta.getxSize()) {
throw new IllegalArgumentException("range invalid");
}
this.xRange = range;
return this;
}
@Override
public GridDataFetcher setY(int y) {
return setYRange(new Range(y, y+1));
}
@Override
public GridDataFetcher setYRange(Range range) {
if (range.getStart() < 0 || range.getEnd() > meta.getySize()) {
throw new IllegalArgumentException("range invalid");
}
this.yRange = range;
return this;
}
@Override
public GridDataFetcher setOriginShape(int[] origin, int[] shape) {
if (origin.length != 4 || shape.length != 4) {
throw new IllegalArgumentException("the length of origin and shape must be 4");
}
setTRange(new Range(origin[0], origin[0] + shape[0]));
setZRange(new Range(origin[1], origin[1] + shape[1]));
setXRange(new Range(origin[2], origin[2] + shape[2]));
setYRange(new Range(origin[3], origin[3] + shape[3]));
return this;
}
public int[] getOrigin() {
return new int[] {tRange.getStart(), zRange.getStart(), xRange.getStart(), yRange.getStart()};
}
public int[] getShape() {
return new int[] {tRange.getSize(), zRange.getSize(), xRange.getSize(), yRange.getSize()};
}
private long calcDataSize(int variableCount) {
long dataSize = variableCount;
return dataSize * meta.getDataType().getSize() * tRange.getSize() * zRange.getSize() * xRange.getSize() * yRange.getSize();
}
private List<String> getColumnsToGet() {
if (!meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) {
throw new IllegalArgumentException("unsupported store type");
}
Plane plane = new Plane(new Range(meta.getxSize()), new Range(meta.getySize()));
Plane subPlane = new Plane(xRange, yRange);
if (plane.equals(subPlane)) {
return null;
}
List<String> columnsToGet = new ArrayList<String>();
List<Point> points = BlockUtil.calcBlockPointsCanCoverSubPlane(plane, subPlane,
meta.getStoreOptions().getxSplitCount(), meta.getStoreOptions().getySplitCount());
for (Point point : points) {
columnsToGet.add(String.format(DATA_BLOCK_COL_NAME_FORMAT, point.getX(), point.getY()));
}
return columnsToGet;
}
private void addTask(final AtomicInteger counter, final byte[] buffer, final int pos, String variable, int t, int z, final CountDownLatch latch, final Queue<Exception> exceptions) {
GetDataParam param = new GetDataParam(tableName, meta.getGridDataSetId(), variable, t, z, getColumnsToGet());
asyncClient.getRow(RequestBuilder.buildGetDataRequest(param), new TableStoreCallback<GetRowRequest, GetRowResponse>() {
@Override
public void onCompleted(GetRowRequest req, GetRowResponse res) {
try {
if (res.getRow() == null) {
exceptions.add(new RuntimeException("the row in not exist, pk: " + req.getRowQueryCriteria().getPrimaryKey()));
}
RowParser.parseGridFromRow(res.getRow(), new Plane(xRange, yRange), meta, buffer, pos);
counter.incrementAndGet();
} catch (Exception ex) {
exceptions.add(ex);
} finally {
latch.countDown();
}
}
@Override
public void onFailed(GetRowRequest req, Exception ex) {
try {
exceptions.add(ex);
} finally {
latch.countDown();
}
}
});
}
@Override
public GridDataSet fetch() throws Exception {
long totalFetchDataSize = calcDataSize(variables.size());
if (totalFetchDataSize == 0) {
throw new RuntimeException("no data to fetch");
}
if (totalFetchDataSize > dataSizeLimitForFetch) {
throw new RuntimeException("exceed the max data limit for fetch");
}
GridDataSet dataSet = new GridDataSet(meta);
CountDownLatch latch = new CountDownLatch(variables.size() * tRange.getSize() * zRange.getSize());
Queue<Exception> exceptions = new ConcurrentLinkedQueue<Exception>();
AtomicInteger counter = new AtomicInteger();
int taskCount = 0;
for (String variable : variables) {
int dataSize = (int) calcDataSize(1);
byte[] data = new byte[dataSize];
ByteBuffer buffer = ByteBuffer.wrap(data).asReadOnlyBuffer();
dataSet.addVariable(variable, new Grid4D(buffer, meta.getDataType(), getOrigin(), getShape()));
int curPos = 0;
for (int t = tRange.getStart(); t < tRange.getEnd(); t++) {
for (int z = zRange.getStart(); z < zRange.getEnd(); z++) {
addTask(counter, data, curPos, variable, t, z, latch, exceptions);
curPos += xRange.getSize() * yRange.getSize() * meta.getDataType().getSize();
taskCount++;
}
}
}
latch.await();
if (!exceptions.isEmpty()) {
throw exceptions.peek();
}
if (counter.get() != taskCount) {
throw new RuntimeException("not all task success");
}
return dataSet;
}
}

98
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataWriter.java
Unescape Escape View File

@ -0,0 +1,98 @@
package com.aliyun.tablestore.grid.core;
import com.alicloud.openservices.tablestore.TableStoreWriter;
import com.alicloud.openservices.tablestore.model.*;
import com.aliyun.tablestore.grid.GridDataWriter;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Plane;
import com.aliyun.tablestore.grid.utils.BlockUtil;
import ucar.ma2.InvalidRangeException;
import java.util.ArrayList;
import java.util.List;
import static com.aliyun.tablestore.grid.consts.Constants.*;
public class TableStoreDataWriter implements GridDataWriter {
private final String tableName;
private final GridDataSetMeta meta;
private final TableStoreWriter writer;
public TableStoreDataWriter(TableStoreWriter writer, String tableName, GridDataSetMeta dataSetMeta) {
this.writer = writer;
this.tableName = tableName;
this.meta = dataSetMeta;
}
private void checkDataSize(String variable, int t, int z, Grid2D grid2D) {
if (!meta.getVariables().contains(variable)) {
throw new IllegalArgumentException("The data set dose not include this variable: " + variable);
}
if (t >= meta.gettSize()) {
throw new IllegalArgumentException("t must be in range: [0, " + meta.gettSize() + ")");
}
if (z >= meta.getzSize()) {
throw new IllegalArgumentException("z must be in range: [0, " + meta.getzSize() + ")");
}
Plane plane = new Plane(grid2D.getOrigin(), grid2D.getShape());
if (plane.getxRange().getStart() != 0 || plane.getyRange().getStart() != 0) {
throw new IllegalArgumentException("xStart and yStart in grid2D must be 0");
}
if (plane.getxRange().getSize() != meta.getxSize()) {
throw new IllegalArgumentException("xSize in grid2D must be equal to gridDataSetMeta's xSize");
}
if (plane.getyRange().getSize() != meta.getySize()) {
throw new IllegalArgumentException("ySize in grid2D must be equal to gridDataSetMeta's ySize");
}
}
private List<Column> splitDataToColumns(Grid2D grid2D) throws InvalidRangeException {
List<Column> columns = new ArrayList<Column>();
List<Grid2D> blocks = BlockUtil.splitGrid2DToBlocks(grid2D, meta.getStoreOptions().getxSplitCount(), meta.getStoreOptions().getySplitCount());
for (Grid2D block : blocks) {
columns.add(new Column(String.format(DATA_BLOCK_COL_NAME_FORMAT, block.getPlane().getxRange().getStart(),
block.getPlane().getyRange().getStart()), ColumnValue.fromBinary(block.getDataAsByteArray())));
}
return columns;
}
private void writeToTableStore(String variable, int t, int z, List<Column> columns) {
if (columns.size() == 0) {
throw new IllegalArgumentException("columns are empty");
}
PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId()));
builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(variable));
builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(t));
builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(z));
PrimaryKey pk = builder.build();
RowUpdateChange rowUpdateChange = new RowUpdateChange(tableName, pk);
int currentSize = 0;
for (int i = 0; i < columns.size(); i++) {
if (currentSize != 0 && (currentSize + columns.get(i).getDataSize()) > MAX_REQUEST_SIZE) {
writer.addRowChange(rowUpdateChange);
rowUpdateChange = new RowUpdateChange(tableName, pk);
currentSize = 0;
}
rowUpdateChange.put(columns.get(i));
currentSize += columns.get(i).getDataSize();
}
writer.addRowChange(rowUpdateChange);
writer.flush();
}
@Override
public void writeGrid2D(String variable, int t, int z, Grid2D grid2D) throws Exception {
checkDataSize(variable, t, z, grid2D);
if (meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) {
List<Column> columns = splitDataToColumns(grid2D);
writeToTableStore(variable, t, z, columns);
} else {
throw new IllegalArgumentException("unsupported store type");
}
}
}

20
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/DeleteDataParam.java
Unescape Escape View File

@ -0,0 +1,20 @@
package com.aliyun.tablestore.grid.model;
import lombok.Data;
@Data
public class DeleteDataParam {
private String dataTableName;
private String dataSetId;
private String variable;
private int t;
private int z;
public DeleteDataParam(String dataTableName, String dataSetId, String variable, int t, int z) {
this.dataTableName = dataTableName;
this.dataSetId = dataSetId;
this.variable = variable;
this.t = t;
this.z = z;
}
}

70
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GetDataParam.java
Unescape Escape View File

@ -0,0 +1,70 @@
package com.aliyun.tablestore.grid.model;
import java.util.List;
public class GetDataParam {
private String dataTableName;
private String dataSetId;
private String variable;
private int t;
private int z;
private List<String> columnsToGet;
public GetDataParam(String dataTableName, String dataSetId, String variable, int t, int z, List<String> columnsToGet) {
this.dataTableName = dataTableName;
this.dataSetId = dataSetId;
this.variable = variable;
this.t = t;
this.z = z;
this.columnsToGet = columnsToGet;
}
public String getDataTableName() {
return dataTableName;
}
public void setDataTableName(String dataTableName) {
this.dataTableName = dataTableName;
}
public String getDataSetId() {
return dataSetId;
}
public void setDataSetId(String dataSetId) {
this.dataSetId = dataSetId;
}
public String getVariable() {
return variable;
}
public void setVariable(String variable) {
this.variable = variable;
}
public int getT() {
return t;
}
public void setT(int t) {
this.t = t;
}
public int getZ() {
return z;
}
public void setZ(int z) {
this.z = z;
}
public List<String> getColumnsToGet() {
return columnsToGet;
}
public void setColumnsToGet(List<String> columnsToGet) {
this.columnsToGet = columnsToGet;
}
}

42
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSet.java
Unescape Escape View File

@ -0,0 +1,42 @@
package com.aliyun.tablestore.grid.model;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class GridDataSet {
private final GridDataSetMeta meta;
private final Map<String, Grid4D> variables;
public GridDataSet(GridDataSetMeta meta) {
this.meta = meta;
this.variables = new ConcurrentHashMap<String, Grid4D>();
}
public GridDataSet(GridDataSetMeta meta, Map<String, Grid4D> variables) {
this.meta = meta;
this.variables = variables;
}
public void addVariable(String variable, Grid4D grid4D) {
this.variables.put(variable, grid4D);
}
public Grid4D getVariable(String variable) {
if (variables == null) {
return null;
}
return variables.get(variable);
}
public Map<String, Grid4D> getVariables() {
return variables;
}
public GridDataSetMeta getMeta() {
return meta;
}
}

124
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSetMeta.java
Unescape Escape View File

@ -0,0 +1,124 @@
package com.aliyun.tablestore.grid.model;
import ucar.ma2.DataType;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class GridDataSetMeta {
private String gridDataSetId;
private final DataType dataType;
private List<String> variables;
private int tSize;
private int zSize;
private int xSize;
private int ySize;
private List<String> forecastHours;
private final StoreOptions storeOptions;
private Map<String, Object> attributes;
public GridDataSetMeta(String gridDataSetId, DataType dataType, List<String> variables, int tSize, int zSize, int xSize, int ySize, StoreOptions storeOptions) {
assert gridDataSetId != null;
assert variables != null;
assert storeOptions != null;
this.gridDataSetId = gridDataSetId;
this.dataType = dataType;
this.variables = variables;
this.tSize = tSize;
this.zSize = zSize;
this.xSize = xSize;
this.ySize = ySize;
this.storeOptions = storeOptions;
}
public String getGridDataSetId() {
return gridDataSetId;
}
public void setGridDataSetId(String gridDataSetId) {
this.gridDataSetId = gridDataSetId;
}
public List<String> getVariables() {
return variables;
}
public void setVariables(List<String> variables) {
this.variables = variables;
}
public int gettSize() {
return tSize;
}
public void settSize(int tSize) {
this.tSize = tSize;
}
public int getzSize() {
return zSize;
}
public void setzSize(int zSize) {
this.zSize = zSize;
}
public int getxSize() {
return xSize;
}
public void setxSize(int xSize) {
this.xSize = xSize;
}
public int getySize() {
return ySize;
}
public void setySize(int ySize) {
this.ySize = ySize;
}
public Map<String, Object> getAttributes() {
return attributes;
}
public void setAttributes(Map<String, Object> attributes) {
assert attributes != null;
for (String key : attributes.keySet()) {
if (key.startsWith("_")) {
throw new IllegalArgumentException("attribute key can't start with \"_\"");
}
}
this.attributes = attributes;
}
public void addAttribute(String key, Object value) {
if (key.startsWith("_")) {
throw new IllegalArgumentException("attribute key can't start with \"_\"");
}
if (this.attributes == null) {
this.attributes = new ConcurrentHashMap<String, Object>();
}
this.attributes.put(key, value);
}
public StoreOptions getStoreOptions() {
return storeOptions;
}
public DataType getDataType() {
return dataType;
}
public void setForecastHours(List<String> forecastHours) {
this.forecastHours = forecastHours;
}
public List<String> getForecastHours() {
return forecastHours;
}
}

43
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryGridDataSetResult.java
Unescape Escape View File

@ -0,0 +1,43 @@
package com.aliyun.tablestore.grid.model;
import java.util.List;
public class QueryGridDataSetResult {
private List<GridDataSetMeta> gridDataSetMetas;
private long totalCount;
private boolean isAllSuccess;
private byte[] nextToken;
public List<GridDataSetMeta> getGridDataSetMetas() {
return gridDataSetMetas;
}
public void setGridDataSetMetas(List<GridDataSetMeta> gridDataSetMetas) {
this.gridDataSetMetas = gridDataSetMetas;
}
public long getTotalCount() {
return totalCount;
}
public void setTotalCount(long totalCount) {
this.totalCount = totalCount;
}
public boolean isAllSuccess() {
return isAllSuccess;
}
public void setAllSuccess(boolean allSuccess) {
isAllSuccess = allSuccess;
}
public byte[] getNextToken() {
return nextToken;
}
public void setNextToken(byte[] nextToken) {
this.nextToken = nextToken;
}
}

82
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryParams.java
Unescape Escape View File

@ -0,0 +1,82 @@
package com.aliyun.tablestore.grid.model;
import com.alicloud.openservices.tablestore.model.search.sort.Sort;
public class QueryParams {
private Integer offset;
private Integer limit;
private Sort sort;
private byte[] token;
private boolean getTotalCount = false;
public QueryParams() {
}
public QueryParams(int limit) {
this.limit = limit;
}
public QueryParams(int offset, int limit) {
this.offset = offset;
this.limit = limit;
}
public QueryParams(int offset, int limit, Sort sort) {
this.offset = offset;
this.limit = limit;
this.sort = sort;
}
public QueryParams(byte[] token, int limit) {
this.token = token;
this.limit = limit;
}
public Integer getOffset() {
return offset;
}
public QueryParams setOffset(Integer offset) {
this.offset = offset;
return this;
}
public Integer getLimit() {
return limit;
}
public QueryParams setLimit(Integer limit) {
this.limit = limit;
return this;
}
public Sort getSort() {
return sort;
}
public QueryParams setSort(Sort sort) {
this.sort = sort;
return this;
}
public byte[] getToken() {
return token;
}
public QueryParams setToken(byte[] token) {
this.token = token;
return this;
}
public boolean isGetTotalCount() {
return getTotalCount;
}
public QueryParams setGetTotalCount(boolean getTotalCount) {
this.getTotalCount = getTotalCount;
return this;
}
}

41
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/StoreOptions.java
Unescape Escape View File

@ -0,0 +1,41 @@
package com.aliyun.tablestore.grid.model;
public class StoreOptions {
public enum StoreType {
SLICE
}
private StoreType storeType;
private int xSplitCount = 10;
private int ySplitCount = 10;
public StoreOptions(StoreType storeType) {
this.storeType = storeType;
}
public StoreType getStoreType() {
return storeType;
}
public void setStoreType(StoreType storeType) {
this.storeType = storeType;
}
public int getxSplitCount() {
return xSplitCount;
}
public void setxSplitCount(int xSplitCount) {
this.xSplitCount = xSplitCount;
}
public int getySplitCount() {
return ySplitCount;
}
public void setySplitCount(int ySplitCount) {
this.ySplitCount = ySplitCount;
}
}

58
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid.java
Unescape Escape View File

@ -0,0 +1,58 @@
package com.aliyun.tablestore.grid.model.grid;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import java.nio.ByteBuffer;
public abstract class Grid {
protected ByteBuffer buffer;
protected DataType dataType;
protected int[] origin;
protected int[] shape;
public Grid(ByteBuffer buffer, DataType dataType, int[] origin, int[] shape) {
this.buffer = buffer;
this.dataType = dataType;
this.origin = origin;
this.shape = shape;
int size = dataType.getSize();
for (int i = 0; i < shape.length; i++) {
size *= shape[i];
}
if (buffer.remaining() != size) {
throw new IllegalArgumentException("data length and shape mismatch");
}
if (origin.length != shape.length) {
throw new IllegalArgumentException("the length of origin and shape mismatch");
}
}
public int getDataSize() {
return buffer.remaining();
}
public byte[] getDataAsByteArray() {
byte[] data = new byte[getDataSize()];
buffer.duplicate().get(data);
return data;
}
public int[] getOrigin() {
return origin;
}
public int[] getShape() {
return shape;
}
public DataType getDataType() {
return dataType;
}
public Array toArray() {
Array array = Array.factory(dataType, shape, buffer.duplicate());
return array;
}
}

19
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid2D.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.aliyun.tablestore.grid.model.grid;
import ucar.ma2.DataType;
import java.nio.ByteBuffer;
public class Grid2D extends Grid {
public Grid2D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) {
super(data, dataType, origin, shape);
if (origin.length != 2 || shape.length != 2) {
throw new IllegalArgumentException("the length of origin and shape must be 2");
}
}
public Plane getPlane() {
return new Plane(getOrigin(), getShape());
}
}

30
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid3D.java
Unescape Escape View File

@ -0,0 +1,30 @@
package com.aliyun.tablestore.grid.model.grid;
import ucar.ma2.DataType;
import java.nio.ByteBuffer;
import java.util.Arrays;
public class Grid3D extends Grid {
public Grid3D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) {
super(data, dataType, origin, shape);
if (origin.length != 3 || shape.length != 3) {
throw new IllegalArgumentException("the length of origin and shape must be 2");
}
}
public Grid2D getGrid2D(int idx) {
if (idx < 0 || idx >= shape[0]) {
throw new IllegalArgumentException("index out of range");
}
int itemSize = shape[1] * shape[2] * dataType.getSize();
int pos = idx * itemSize;
ByteBuffer newBuffer = buffer.slice();
newBuffer.position(pos);
newBuffer.limit(pos + itemSize);
newBuffer = newBuffer.slice();
return new Grid2D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length),
Arrays.copyOfRange(shape, 1, shape.length));
}
}

30
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid4D.java
Unescape Escape View File

@ -0,0 +1,30 @@
package com.aliyun.tablestore.grid.model.grid;
import ucar.ma2.DataType;
import java.nio.ByteBuffer;
import java.util.Arrays;
public class Grid4D extends Grid {
public Grid4D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) {
super(data, dataType, origin, shape);
if (origin.length != 4 || shape.length != 4) {
throw new IllegalArgumentException("the length of origin and shape must be 2");
}
}
public Grid3D getGrid3D(int idx) {
if (idx < 0 || idx >= shape[0]) {
throw new IllegalArgumentException("index out of range");
}
int itemSize = shape[1] * shape[2] * shape[3] * dataType.getSize();
int pos = idx * itemSize;
ByteBuffer newBuffer = buffer.slice();
newBuffer.position(pos);
newBuffer.limit(pos + itemSize);
newBuffer = newBuffer.slice();
return new Grid3D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length),
Arrays.copyOfRange(shape, 1, shape.length));
}
}

55
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Plane.java
Unescape Escape View File

@ -0,0 +1,55 @@
package com.aliyun.tablestore.grid.model.grid;
public class Plane {
private Range xRange;
private Range yRange;
public Plane(Range xRange, Range yRange) {
this.xRange = xRange;
this.yRange = yRange;
}
public Plane(int[] origin, int[] shape) {
if (origin.length != 2 || shape.length != 2) {
throw new IllegalArgumentException("the length of origin and shape must be 2");
}
this.xRange = new Range(origin[0], origin[0] + shape[0]);
this.yRange = new Range(origin[1], origin[1] + shape[1]);
}
public int[] getOrigin() {
return new int[] {xRange.getStart(), yRange.getStart()};
}
public int[] getShape() {
return new int[] {xRange.getSize(), yRange.getSize()};
}
public Range getxRange() {
return xRange;
}
public void setxRange(Range xRange) {
this.xRange = xRange;
}
public Range getyRange() {
return yRange;
}
public void setyRange(Range yRange) {
this.yRange = yRange;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o instanceof Plane) {
return xRange.equals(((Plane) o).getxRange()) && yRange.equals(((Plane) o).getyRange());
}
return false;
}
}

39
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Point.java
Unescape Escape View File

@ -0,0 +1,39 @@
package com.aliyun.tablestore.grid.model.grid;
public class Point {
private int x;
private int y;
public Point(int x, int y) {
this.x = x;
this.y = y;
}
public int getX() {
return x;
}
public void setX(int x) {
this.x = x;
}
public int getY() {
return y;
}
public void setY(int y) {
this.y = y;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o instanceof Point) {
return x == ((Point) o).x && y == ((Point) o).y;
}
return false;
}
}

53
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Range.java
Unescape Escape View File

@ -0,0 +1,53 @@
package com.aliyun.tablestore.grid.model.grid;
public class Range {
private int start;
private int end;
public Range(int end) {
this.start = 0;
this.end = end;
}
public Range(int start, int end) {
this.start = start;
this.end = end;
}
public int getStart() {
return start;
}
public void setStart(int start) {
this.start = start;
}
public int getEnd() {
return end;
}
public void setEnd(int end) {
this.end = end;
}
public int getSize() {
return this.end - this.start;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o instanceof Range) {
return start == ((Range) o).start && end == ((Range) o).end;
}
return false;
}
@Override
public String toString() {
return ("[" + start + ", " + end + ")");
}
}

90
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/BlockUtil.java
Unescape Escape View File

@ -0,0 +1,90 @@
package com.aliyun.tablestore.grid.utils;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.aliyun.tablestore.grid.model.grid.Plane;
import com.aliyun.tablestore.grid.model.grid.Point;
import com.aliyun.tablestore.grid.model.grid.Range;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import ucar.ma2.InvalidRangeException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
public class BlockUtil {
public static List<Grid2D> splitGrid2DToBlocks(Grid2D grid2D, int xSplitCount, int ySplitCount) throws InvalidRangeException {
Array array = grid2D.toArray();
int blockXSize = (grid2D.getPlane().getxRange().getSize() - 1) / xSplitCount + 1;
int blockYSize = (grid2D.getPlane().getyRange().getSize() - 1) / ySplitCount + 1;
List<Grid2D> result = new ArrayList<Grid2D>();
for (int i = 0; i < xSplitCount; i++) {
int startX = i * blockXSize;
int endX = Math.min(grid2D.getPlane().getxRange().getSize(), startX + blockXSize);
if (startX >= grid2D.getPlane().getxRange().getSize()) {
break;
}
for (int j = 0; j < ySplitCount; j++) {
int startY = j * blockYSize;
int endY = Math.min(grid2D.getPlane().getyRange().getSize(), startY + blockYSize);
if (startY >= grid2D.getPlane().getyRange().getSize()) {
break;
}
int[] origin = new int[] { startX, startY };
int[] shape = new int[] { endX - startX, endY - startY };
Array section = array.section(origin, shape);
Grid2D block = new Grid2D(section.getDataAsByteBuffer(), grid2D.getDataType(), origin, shape);
result.add(block);
}
}
return result;
}
public static List<Point> calcBlockPointsCanCoverSubPlane(Plane plane, Plane subPlane, int xSplitCount, int ySplitCount) {
int blockXSize = (plane.getxRange().getSize() - 1) / xSplitCount + 1;
int blockYSize = (plane.getyRange().getSize() - 1) / ySplitCount + 1;
Range xBlockIndexRange = new Range(
subPlane.getxRange().getStart() / blockXSize,
(subPlane.getxRange().getEnd() - 1) / blockXSize + 1);
Range yBlockIndexRange = new Range(
subPlane.getyRange().getStart() / blockYSize,
(subPlane.getyRange().getEnd() - 1) / blockYSize + 1);
List<Point> points = new ArrayList<Point>();
for (int xIdx = xBlockIndexRange.getStart(); xIdx < xBlockIndexRange.getEnd(); xIdx++) {
for (int yIdx = yBlockIndexRange.getStart(); yIdx < yBlockIndexRange.getEnd(); yIdx++) {
Point point = new Point(xIdx * blockXSize, yIdx * blockYSize);
points.add(point);
}
}
return points;
}
public static Grid2D buildGrid2DFromBlocks(Plane plane, DataType dataType, List<Grid2D> blocks, byte[] buffer, int pos) {
int size = plane.getxRange().getSize() * plane.getyRange().getSize() * dataType.getSize();
if (buffer.length - pos < size) {
throw new IllegalArgumentException("buffer not enough");
}
int count = 0;
for (Grid2D block : blocks) {
Plane blockPlane = block.getPlane();
for (int x = Math.max(blockPlane.getxRange().getStart(), plane.getxRange().getStart());
x < Math.min(blockPlane.getxRange().getEnd(), plane.getxRange().getEnd()); x++) {
for (int y = Math.max(blockPlane.getyRange().getStart(), plane.getyRange().getStart());
y < Math.min(blockPlane.getyRange().getEnd(), plane.getyRange().getEnd()); y++) {
int posInBlock = dataType.getSize() * ((x - blockPlane.getxRange().getStart()) * (blockPlane.getyRange().getSize()) + (y - blockPlane.getyRange().getStart())) ;
int posInData = dataType.getSize() * ((x - plane.getxRange().getStart()) * plane.getyRange().getSize() + (y - plane.getyRange().getStart()));
System.arraycopy(block.getDataAsByteArray(), posInBlock, buffer, pos + posInData, dataType.getSize());
count += dataType.getSize();
}
}
}
if (count != size) {
throw new RuntimeException("the blocks does not contain enough data");
}
ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, pos, size);
return new Grid2D(byteBuffer, dataType, plane.getOrigin(), plane.getShape());
}
}

47
tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/ValueUtil.java
Unescape Escape View File

@ -0,0 +1,47 @@
package com.aliyun.tablestore.grid.utils;
import com.alicloud.openservices.tablestore.model.ColumnValue;
public class ValueUtil {
public static ColumnValue toColumnValue(Object value) {
if (value instanceof Long) {
return ColumnValue.fromLong((Long) value);
} else if (value instanceof Integer) {
return ColumnValue.fromLong(((Integer) value).longValue());
} else if (value instanceof Double) {
return ColumnValue.fromDouble((Double) value);
} else if (value instanceof String) {
return ColumnValue.fromString((String) value);
} else if (value instanceof Boolean) {
return ColumnValue.fromBoolean((Boolean) value);
} else if (value instanceof byte[]) {
return ColumnValue.fromBinary((byte[]) value);
} else {
throw new IllegalArgumentException("unsupported type: " + value.getClass());
}
}
public static Object toObject(ColumnValue value) {
switch (value.getType()) {
case INTEGER: {
return value.asLong();
}
case STRING: {
return value.asString();
}
case BOOLEAN: {
return value.asBoolean();
}
case DOUBLE: {
return value.asDouble();
}
case BINARY: {
return value.asBinary();
}
default: {
throw new RuntimeException("unexpected");
}
}
}
}

13
tablestore-grid-master/src/main/resources/META-INF/MANIFEST.MF
Unescape Escape View File

@ -0,0 +1,13 @@
Manifest-Version: 1.0
Class-Path: httpservices-5.5.3.jar re2j-1.3.jar udunits-5.5.3.jar jsr305
-3.0.2.jar tablestore-5.16.0-jar-with-dependencies.jar cdm-core-5.5.3.j
ar j2objc-annotations-1.3.jar disruptor-3.4.4.jar logback-core-1.2.12.j
ar flatbuffers-java-1.11.0.jar jcommander-1.78.jar httpcore-4.4.13.jar
slf4j-api-1.7.28.jar listenablefuture-9999.0-empty-to-avoid-conflict-wi
th-guava.jar joda-time-2.10.3.jar lombok-1.18.30.jar commons-io-2.11.0.
jar commons-codec-1.11.jar protobuf-java-3.19.3.jar jdom2-2.0.6.jar fai
lureaccess-1.0.1.jar checker-qual-3.5.0.jar jj2000-5.4.jar httpclient-4
.5.13.jar commons-logging-1.2.jar guava-30.1-jre.jar gson-2.8.5.jar err
or_prone_annotations-2.3.4.jar logback-classic-1.2.12.jar httpmime-4.5.
13.jar grib-5.5.3.jar

21
tablestore-grid-master/src/main/resources/logback.xml
Unescape Escape View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan = "true" debug="false">
<!-- 控制台日志输出-->
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n</pattern>
<!-- <pattern>${log.pattern}</pattern> -->
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>debug</level>
</filter>
</appender>
<!-- 定义某个包的日志级别 -->
<logger name="com" level="debug"> <!-- 添加控制台输出 -->
<appender-ref ref="console"/>
</logger>
<logger name="com.alicloud.openservices" level="WARN"/>
<logger name="com.aliyun.ots" level="WARN"/>
</configuration>

60
tablestore-grid-master/src/test/java/TestReadGrib2ByNC.java
Unescape Escape View File

@ -0,0 +1,60 @@
import ucar.ma2.Array;
import ucar.ma2.Index;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
import ucar.nc2.Variable;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
/**
* Unit test for simple App.
*/
public class TestReadGrib2ByNC {
public static void main(String[] args){
// String ncFile = "surface.nc";
String file = "C:\\Users\\shi_y\\Desktop\\java_learn\\data_download\\GFSData\\UTC-20230910\\BJT-20230911-0200.grib2";
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
// 打印文件相关信息
// System.out.println(ncfile.toString());
// 获取时间变量
Variable timeVar = ncFile.findVariable("time");
String units = timeVar.findAttribute("units").getStringValue();
System.out.println(units);
// 读取时间变量 surface.nc的单位是"hours since 1900-01-01 00:00:00.0"
System.out.println(timeVar.getFileTypeId());
double[] timeArray = (double[]) timeVar.read().copyTo1DJavaArray();
DateTimeFormatter unitsFormat = DateTimeFormatter.ofPattern("'Hour since 'yyyy-MM-dd'T'HH:mm:ss'Z'");
LocalDateTime unitsDateTime = LocalDateTime.parse(units, unitsFormat).plusHours((long) timeArray[0]);
System.out.println(unitsDateTime);
// 指定变量名读取变量
Variable t2m = ncFile.findVariable("Temperature_isobaric");
// 打印变量的相关信息
System.out.println(t2m.toString());
if (t2m != null) {
System.out.println(t2m.findDimensionIndex("lon"));
}
// 读取所有的数据
assert t2m != null;
Array v = t2m.read("0, 0, 10:20, 20:30").reduce(0).reduce(0);
Index index = v.getIndex(); // 用于跟踪当前索引的位置
index = index.set(1);
System.out.println(v.getShort(index));
} catch (IOException ioe) {
// Handle less-cool exceptions here
throw new RuntimeException(ioe);
} catch (InvalidRangeException e) {
throw new RuntimeException(e);
}
}
}

159
weather-service/pom.xml
Unescape Escape View File

@ -0,0 +1,159 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.htfp.weather</groupId>
<artifactId>weather-service</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>weather-service</name>
<description>weather-service</description>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.6.13</spring-boot.version>
</properties>
<repositories>
<repository>
<id>unidata-all</id>
<name>Unidata All</name>
<url>https://artifacts.unidata.ucar.edu/repository/unidata-all/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>edu.ucar</groupId>
<artifactId>cdm-core</artifactId>
<version>5.5.3</version>
</dependency>
<dependency>
<groupId>edu.ucar</groupId>
<artifactId>grib</artifactId>
<version>5.5.3</version>
</dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>tablestore</artifactId>
<version>5.16.0</version>
<classifier>jar-with-dependencies</classifier>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ai.djl</groupId>
<artifactId>api</artifactId>
<version>0.27.0</version>
</dependency>
<dependency>
<groupId>ai.djl.pytorch</groupId>
<artifactId>pytorch-engine</artifactId>
<version>0.27.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.0</version>
<!-- 去掉Android的包 -->
<exclusions>
<exclusion>
<groupId>com.google.android</groupId>
<artifactId>android</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.tablestore</groupId>
<artifactId>tablestore-grid</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<!-- springmvc的参数valid校验依赖 开始 -->
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator</artifactId>
<version>6.0.16.Final</version>
</dependency>
<!-- springmvc的参数valid校验依赖 结束 -->
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<mainClass>com.htfp.weather.WeatherServiceApplication</mainClass>
<skip>true</skip>
</configuration>
<executions>
<execution>
<id>repackage</id>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

19
weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.htfp.weather;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@SpringBootApplication
public class WeatherServiceApplication {
public static void main(String[] args) {
try {
SpringApplication.run(WeatherServiceApplication.class, args);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

20
weather-service/src/main/java/com/htfp/weather/config/Config.java
Unescape Escape View File

@ -0,0 +1,20 @@
package com.htfp.weather.config;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigBean;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
/**
* @Author : shiyi
* @Date : 2024/1/22 19:02
* @Description :
*/
@Component("config")
public class Config {
@Resource
TableConfigBean tableConfigBean;
}

51
weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
Unescape Escape View File

@ -0,0 +1,51 @@
package com.htfp.weather.download;
import lombok.Data;
import lombok.ToString;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.util.List;
/**
* @author shiyi
*/
@Data
@ToString
public abstract class BaseDataDownloader {
/**起始时间，北京时UTC+8，格式示例：20230901T08:00*/
private String startTimeStr;
private LocalDateTime startTime;
private String refTimeStr;
/** 数据的实际起报UTC时间 */
private OffsetDateTime refTime;
/**
* 获取所有下载文件的相关信息
*/
public abstract List<FileInfo> getFilesInfo();
/**
* 单个文件下载
* @param fileInfo: 目标文件的下载信息
* @return 下载成功/失败
* @throws IOException
*/
public abstract boolean download(FileInfo fileInfo) throws IOException;
/**
* 下载所有目标文件
*
* @param filesInfo
* @return
*/
public abstract boolean downloadAll(List<FileInfo> fileInfoList);
public BaseDataDownloader() {
}
}

31
weather-service/src/main/java/com/htfp/weather/download/FileInfo.java
Unescape Escape View File

@ -0,0 +1,31 @@
package com.htfp.weather.download;
import lombok.Data;
import lombok.ToString;
/**
* @author shiyi
*/
@Data
@ToString
public class FileInfo {
/**保存文件名*/
private String fileName;
/**数据起报时间*/
private String refTimeStr;
/**预报时效*/
private Integer forecastHour;
/**预报时效对应的北京时间*/
private String forecastBJTimeStr;
/**预报时效对应的UTC时间*/
private String forecastUTCTimeStr;
/**下载链接*/
private String url;
private String savePath;
public FileInfo() {
}
}

154
weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java
Unescape Escape View File

@ -0,0 +1,154 @@
package com.htfp.weather.download;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.utils.JSONUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Objects;
/**
* @Author : shiyi
* @Date : 2024/2/28 13:53
* @Description : GFS数据下载配置
*/
@Slf4j @Data
@Component
public class GfsDataConfig {
@NotNull(message = "下载范围 (duration) 不能为空")
@Min(value = 1, message = "下载范围 (duration) 需大于等于1小时")
@Max(value = 36, message = "下载范围 (duration) 不能超过36小时")
private int duration = 5;
@NotNull(message = "最小经度不能为空")
@Min(value = -180, message = "经度最小值为-180（西经180°）")
@Max(value = 180, message = "经度最大值为180（东经180°）")
private Double minLon;
@NotNull(message = "最大经度不能为空")
@Min(value = -180, message = "经度最小值为-180（西经180°）")
@Max(value = 180, message = "经度最大值为180（东经180°）")
private Double maxLon;
@NotNull(message = "最小纬度不能为空")
@Min(value = -90, message = "纬度最小值为-90（南纬90°）")
@Max(value = 90, message = "纬度最大值为90（北纬90°）")
private Double minLat;
@NotNull(message = "最大纬度不能为空")
@Min(value = -90, message = "纬度最小值为-90（南纬90°）")
@Max(value = 90, message = "纬度最大值为90（北纬90°）")
private Double maxLat;
@NotNull(message = "分辨率 (resolution) 不能为空")
private Double resolution = 0.25;
@Valid
@NotEmpty(message = "变量列表 (variables) 不能为空")
// @EnumValid(enumClass = GfsDataVariableEnum.class, message = "变量列表 (variables) 必须为 GfsDataVariableEnum 枚举值")
private ArrayList<String> variables;
@Valid
@NotEmpty(message = "气压高度列表 (pressureLevels) 不能为空")
// @EnumValid(enumClass = GfsLevelsEnum.class, message = "变量列表 (pressureLevels) 必须为 GfsLevelsEnum 枚举值")
private int[] pressureLevels;
private int[] heightLevels = new int[]{2, 10, 20, 30, 40, 50, 80, 100};
@NotEmpty(message = "数据存储路径 (saveRoot) 不能为空")
private String saveRoot;
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json";
public void valid() {
// TODO 2024/4/21: 校验
if (minLon >= maxLon) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度");
}
if (minLat >= maxLat) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度");
}
ArrayList<String> wrongVariables = new ArrayList<>();
ArrayList<Integer> wrongLevels = new ArrayList<>();
for (String variable : variables) {
if (!GfsDownloadVariableEnum.contains(variable)) {
wrongVariables.add(variable);
}
}
if (!wrongVariables.isEmpty()) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, " variables = " +wrongVariables + "非 GfsLevelsEnum 枚举值");
}
for (Integer level : pressureLevels) {
if (!GfsLevelsEnum.contains(level)) {
wrongLevels.add(level);
}
}
if (!wrongLevels.isEmpty()) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, " pressureLevels = " +wrongLevels + "非 GfsLevelsEnum 枚举值");
}
}
@PostConstruct
public void init() {
readConfig();
}
public void readConfig() {
try (InputStream f = new FileInputStream(configPath)){
GfsDataConfig gfsDataConfig = JSONUtils.json2pojo(IOUtils.toString(f), GfsDataConfig.class);
this.duration = gfsDataConfig.getDuration();
this.minLon = gfsDataConfig.getMinLon();
this.maxLon = gfsDataConfig.getMaxLon();
this.minLat = gfsDataConfig.getMinLat();
this.maxLat = gfsDataConfig.getMaxLat();
this.resolution = gfsDataConfig.getResolution();
this.variables = gfsDataConfig.getVariables();
this.pressureLevels = gfsDataConfig.getPressureLevels();
this.saveRoot = gfsDataConfig.getSaveRoot();
log.info("[config] 下载配置初始化: {}", configPath);
} catch (NoSuchFileException e) {
log.error("[config] 配置文件{}不存在", configPath, e);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public void writeConfig(GfsDataConfig updateConfig) throws IOException {
writeConfig(updateConfig, configPath);
}
public void writeConfig(GfsDataConfig updateConfig, String filePath) throws IOException {
this.duration = updateConfig.getDuration();
this.minLon = updateConfig.getMinLon();
this.maxLon = updateConfig.getMaxLon();
this.minLat = updateConfig.getMinLat();
this.maxLat = updateConfig.getMaxLat();
this.resolution = updateConfig.getResolution();
this.variables = updateConfig.getVariables();
this.pressureLevels = updateConfig.getPressureLevels();
this.saveRoot = updateConfig.getSaveRoot();
// 如果目录不存在则创建
File destDir = new File(saveRoot);
if (!destDir.exists()) {
boolean mkdirs = destDir.mkdirs();
if (!mkdirs) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "目录创建失败: " + saveRoot);
}
}
JSONUtils.pojo2jsonFile(this, filePath);
log.info("配置文件 {} 更新为: {}", filePath, updateConfig);
}
}

265
weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
Unescape Escape View File

@ -0,0 +1,265 @@
package com.htfp.weather.download;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.HttpClientUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* @author shiyi
*/
@Slf4j
@Component("gfsDownloader")
@DependsOn("gfsDataConfig")
public class GfsDownloader extends BaseDataDownloader {
@Resource
GfsDataConfig gfsDataConfig;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
private int hourDiff;
private OffsetDateTime realUpdateDateTime;
public GfsDownloader(GfsDataConfig gfsDataConfig) {
super();
this.gfsDataConfig = gfsDataConfig;
}
/**
* 初始化下载起始时间
*/
public void iniTimeSetting() {
// 当前时间
this.setStartTime(LocalDateTime.now());
this.setStartTimeStr(LocalDateTime.now().format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING)));
// 以下时间均为UTC时间
OffsetDateTime utcStartTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now());
realUpdateDateTime = getAvailableUpdateDateTime(utcStartTime);
this.setRefTime(realUpdateDateTime);
this.setRefTimeStr(realUpdateDateTime.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)));
// 起报点和实际请求时间的差异
this.hourDiff = (int) Duration.between(realUpdateDateTime, utcStartTime).toHours();
log.info("[GFS Download] 起始为当前本地时间：{}", this.getStartTime());
log.info("[GFS Download] GFS最近更新时间UTC：{}，本地时间: {}", realUpdateDateTime, DateTimeUtils.getLocalZoneDateTime(realUpdateDateTime));
}
@Override
public boolean download(FileInfo fileInfo) {
try {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
if (!destDir.exists()) {
destDir.mkdirs();
}
File fileOut = new File(destDir, fileInfo.getFileName());
// if (fileOut.exists()) {
// log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath());
// return true;
// }
log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
// FileOutputStream fos = new FileOutputStream(filePath);
// try {
// ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream());
// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// } catch (ConnectException e) {
// log.error("连接超时:{}", url);
// }
//
// fos.flush();
// fos.close();
FileUtils.copyURLToFile(new URL(url), fileOut);
log.info("[GFS Download] 文件下载成功: {}", fileOut);
return fileValid(fileOut.getAbsolutePath());
} catch (Exception e) {
log.error("[GFS Download] 文件 {} 下载失败：{}", fileInfo, e.getMessage());
return false;
}
}
private boolean fileValid(String file) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
ncFile.getLocation();
return true;
} catch (IOException e){
return false;
}
}
@Override
public boolean downloadAll(List<FileInfo> fileInfoList) {
log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size());
List<Future<Boolean>> futures = new ArrayList<>();
for (FileInfo fileInfo : fileInfoList) {
futures.add(executorService.submit(() -> download(fileInfo)));
}
boolean allCompleted = true;
for (Future<Boolean> future : futures) {
try {
allCompleted = allCompleted && future.get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
return allCompleted;
}
@Override
public List<FileInfo> getFilesInfo() {
// 时间选项填充
String lonlatBoxStr = null;
String levelsStr = null;
String variablesStr = null;
try {
lonlatBoxStr = fillUrlForLonLat();
levelsStr = fillUrlForLevels();
variablesStr = fillUrlForVariables();
} catch (Exception e) {
log.error("[GFS Download] 文件下载参数生成错误");
throw new RuntimeException(e);
}
// 预报数据时间分辨率
final int forecastStep = 1;
// 分辨率为3h，整数截断；包括目标时刻本身
final int nFiles = 1 + gfsDataConfig.getDuration() / forecastStep;
// 存储文件信息
List<FileInfo> fileInfoList = new ArrayList<>(nFiles);
for (int i = 0; i < nFiles; i++) {
FileInfo fileInfo = new FileInfo();
int forecastHour = i * forecastStep + hourDiff;
fileInfo.setForecastHour(forecastHour);
String baseURL = getBaseURL(forecastHour, levelsStr, variablesStr, lonlatBoxStr);
fileInfo.setUrl(baseURL);
fileInfo.setRefTimeStr(this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)));
fileInfo.setForecastUTCTimeStr(
this.getRefTime().plusHours(fileInfo.getForecastHour())
.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))
);
fileInfo.setForecastBJTimeStr(
DateTimeUtils.getLocalZoneDateTime(this.getRefTime().plusHours(fileInfo.getForecastHour()))
.format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING))
);
fileInfo.setFileName(String.format("%s-from-%s+%d.grib2", fileInfo.getForecastBJTimeStr(), fileInfo.getRefTimeStr(), forecastHour));
fileInfo.setSavePath(gfsDataConfig.getSaveRoot() + "/" + fileInfo.getRefTimeStr());
fileInfoList.add(fileInfo);
}
return fileInfoList;
}
private String getBaseURL(int forcastHour, String levelsStr, String variablesStr, String lonlatBoxStr) {
String baseFormat = "https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25_1hr.pl?dir=/gfs.%s/%02d/atmos&file=gfs.t%02dz.pgrb2.0p25.f%03d";
String realUpdateDateString = realUpdateDateTime.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
int refHour = realUpdateDateTime.getHour();
String baseurl = String.format(baseFormat, realUpdateDateString, refHour, refHour, forcastHour);
StringJoiner stringJoiner = new StringJoiner("&");
return stringJoiner.add(baseurl).add(levelsStr).add(variablesStr).add(lonlatBoxStr).toString();
}
/**
* 获取距离指定时刻最近的更新时刻
*/
private OffsetDateTime getAvailableUpdateDateTime(OffsetDateTime utcStartTime) {
// 获取临近的更新点
int hour = utcStartTime.getHour();
int[] updateHours = {0, 6, 12, 18};
int nearUpdateHour = 0;
for (int i : updateHours) {
if (i < hour) {
nearUpdateHour = i;
}
}
OffsetDateTime nearUpdateDateTime = utcStartTime.withHour(nearUpdateHour).withMinute(0).withSecond(0).withNano(0);
OffsetDateTime now = OffsetDateTime.now();
// 如果最近的更新点还没有更新数据，那么返回前一个更新点
// if (isGfsUpdated(nearUpdateDateTime)) {
if (now.isAfter(nearUpdateDateTime.plusHours(4))) {
// GFS延迟4小时才能获取到数据，比如04时更新00时的数据
return nearUpdateDateTime;
} else {
return nearUpdateDateTime.minusHours(6);
}
}
/**
* 判断GFS是否更新
*
* @return
*/
public boolean isGfsUpdated(OffsetDateTime utcUpdateTime) {
String timeString = utcUpdateTime.format(DateTimeFormatter.ofPattern("'gfs.'yyyyMMdd/HH"));
String url = String.format("https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/%s/", timeString);
try {
String httpString = HttpClientUtils.sendGet(url, null);
if (StringUtils.isEmpty(httpString)) {
return false;
}
// FIXME 2024/5/17: 可能遇到正在更新，文件夹存在，但是文件不一定存在
return httpString.contains("atmos/") && httpString.contains("wave/");
} catch (Exception e) {
return false;
}
}
private String fillUrlForLonLat() {
// 根据传入参数生成对应的URL请求信息，生成字符串
Double topLat = gfsDataConfig.getMaxLat();
Double leftLon = gfsDataConfig.getMinLon();
Double rightLon = gfsDataConfig.getMaxLon();
Double bottomLat = gfsDataConfig.getMinLat();
return String.format("subregion=&toplat=%.2f&leftlon=%.2f&rightlon=%.2f&bottomlat=%.2f", topLat, leftLon, rightLon, bottomLat);
}
private String fillUrlForLevels() {
int[] pressureLevels = gfsDataConfig.getPressureLevels();
if (pressureLevels.length == 0) {
throw (new IllegalArgumentException("至少指定一个层次"));
}
StringJoiner stringJoiner = new StringJoiner("&");
for (int pres : pressureLevels) {
stringJoiner.add(String.format("lev_%d_mb=on", pres));
}
// 添加地面2m和10m的信息
int[] heightLevels = gfsDataConfig.getHeightLevels();
for (int height : heightLevels) {
stringJoiner.add(String.format("lev_%d_m_above_ground", height));
}
return stringJoiner.toString();
}
private String fillUrlForVariables() {
ArrayList<String> variables = gfsDataConfig.getVariables();
StringJoiner stringJoiner = new StringJoiner("&");
if (variables.isEmpty()) {
throw (new IllegalArgumentException("至少指定一个下载变量"));
}
for (String variable : variables) {
stringJoiner.add(String.format("var_%s=on", variable));
}
return stringJoiner.toString();
}
}

54
weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java
Unescape Escape View File

@ -0,0 +1,54 @@
package com.htfp.weather.griddata.common;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* @Author : shiyi
* @Date : 2024/1/15 14:54
* @Description : 文件变量名和表中变量名的映射
*/
public enum GfsVariableNameEnum {
//
TEMP("temp", "Temperature_isobaric"),
CLOUD("cloud", "Total_cloud_cover_isobaric"),
WIND_SPEED("windSpeed", "Wind_speed_isobaric"),
WIND360("wind360", "Wind_direction_isobaric"),
;
private static final ConcurrentHashMap<String, String> variableName = new ConcurrentHashMap<String, String>() {{
put("temp", "Temperature_isobaric");
put("cloud", "Total_cloud_cover_isobaric");
put("windSpeed", "Wind_speed_isobaric");
put("wind360", "Wind_direction_isobaric");
}};
String nameInApi;
String nameInFile;
GfsVariableNameEnum(String nameInApi, String nameInFile) {
this.nameInApi = nameInApi;
this.nameInFile = nameInFile;
}
public String getNameInApi() {
return nameInApi;
}
public String getNameInFile() {
return nameInFile;
}
public static String getGfsVariableName(GfsVariableNameEnum gfsVariableNameEnum) {
return variableName.get(gfsVariableNameEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);
}
public static List<String> getVariableNamesInApi() {
return new ArrayList<>(variableName.keySet());
}
public static List<String> getVariableNamesInFile() {
return new ArrayList<>(variableName.values());
}
}

108
weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java
Unescape Escape View File

@ -0,0 +1,108 @@
package com.htfp.weather.griddata.common;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
import javax.annotation.PostConstruct;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Author : shiyi
* @Date : 2024/1/15 19:10
* @Description : 表格meta配置
*/
@Slf4j @Component
public class TableConfig {
@Autowired
GfsDataConfig dataConfig;
/**
* 表名和index名
*/
public static String DATA_TABLE_NAME;
public static String META_TABLE_NAME;
public static String DATA_INDEX_NAME;
public static String META_INDEX_NAME;
/**
* 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型
*/
public static String DATA_DIR;
public static List<String> VARIABLE_LIST;
public static int lonSize;
public static List<Double> lonList;
public static int latSize;
public static List<Double> latList;
public static int levSize;
public static List<Integer> levList;
public static int timeSizeMax = 72;
public static List<String> timeList;
public static DataType DATA_TYPE = DataType.FLOAT;
// 数据过期时间，单位为秒
public static int TIME_TO_LIVE;
@PostConstruct
private void initTableConfig() {
initDatasetConfig();
initLonList();
initLatList();
initLevList();
}
/**
* 初始化表格配置
*/
public static void initDatasetConfig() {
log.info("init table config...");
String path = Objects.requireNonNull(TableConfig.class.getClassLoader().getResource("config/tableConf.json")).getPath();
try (InputStream f = new FileInputStream(path)){
Gson gson = new Gson();
Map map = gson.fromJson(IOUtils.toString(f), Map.class);
DATA_TABLE_NAME = (String) map.get("dataTableName");
META_TABLE_NAME = (String) map.get("metaTableName");
DATA_INDEX_NAME = (String) map.get("dataIndexName");
META_INDEX_NAME = (String) map.get("metaIndexName");
DATA_DIR = (String) map.get("dataDir");
VARIABLE_LIST = (ArrayList<String>) map.get("variableList");
lonSize = ((Double) map.get("lonSize")).intValue();
latSize = ((Double) map.get("latSize")).intValue();
levSize = ((Double) map.get("levSize")).intValue();
TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
private void initLonList() {
double lonStart = dataConfig.getMinLon();
double res = dataConfig.getResolution();
lonList = new ArrayList<>();
for (int i = 0; i < lonSize; i++) {
lonList.add(lonStart + i * res);
}
}
private void initLatList() {
double latStart = dataConfig.getMinLat();
double res = dataConfig.getResolution();
latList = new ArrayList<>();
for (int i = 0; i < latSize; i++) {
latList.add(latStart + i * res);
}
}
private void initLevList() {
levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList());
}
}

144
weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java
Unescape Escape View File

@ -0,0 +1,144 @@
package com.htfp.weather.griddata.common;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.google.gson.Gson;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.utils.JSONUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ucar.ma2.DataType;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Data @Component @Slf4j
@JsonIgnoreProperties(ignoreUnknown = true)
public class TableConfigBean {
@Resource
@JsonIgnore
GfsDataConfig dataConfig;
@Value("duration")
public String dataTableName;
public String metaTableName;
public String dataIndexName;
public String metaIndexName;
/**
* 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型
*/
public String dataDir;
public List<String> variableList;
public int lonSize;
public int latSize;
public int levSize;
public int timeToLive;
@JsonIgnore
public int timeSizeMax = 99;
@JsonIgnore
public double[] lonList;
@JsonIgnore
public double[] latList;
@JsonIgnore
public int[] levList;
// @JsonIgnore
// public List<Integer> forecastTimeList; // 预报时效列表
@JsonIgnore
public DataType dataType = DataType.FLOAT;
// note 修改作用范围在classes目录下，resource文件下不改变
@JsonIgnore
private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath();
@PostConstruct
private void initTableConfig() {
readConfig();
initLonList();
initLatList();
initLevList();
}
public void readConfig() {
// String pathSeparator = System.getProperty("file.separator");
try (InputStream f = new FileInputStream(configPath)){
String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8);
TableConfigBean tableConfig = JSONUtils.json2pojo(jsonStr, TableConfigBean.class);
this.dataTableName = tableConfig.getDataTableName();
this.metaTableName = tableConfig.getMetaTableName();
this.dataIndexName = tableConfig.getDataIndexName();
this.metaIndexName = tableConfig.getMetaIndexName();
this.variableList = tableConfig.getVariableList();
this.dataDir = tableConfig.getDataDir();
this.lonSize = tableConfig.getLonSize();
this.latSize = tableConfig.getLatSize();
this.levSize = dataConfig.getPressureLevels().length;
this.timeToLive = tableConfig.getTimeToLive();
log.info("[config] 读取 DataTable 配置 : {}", configPath);
} catch (NoSuchFileException e) {
log.error("[config] 配置文件{}不存在", configPath, e);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public void writeConfig(TableConfigBean updateConfig) throws IOException {
writeConfig(updateConfig, configPath);
}
public void writeConfig(TableConfigBean updateConfig, String filePath) throws IOException {
this.dataTableName = updateConfig.getDataTableName();
this.metaTableName = updateConfig.getMetaTableName();
this.dataIndexName = updateConfig.getDataIndexName();
this.metaIndexName = updateConfig.getMetaIndexName();
this.variableList = updateConfig.getVariableList();
this.dataDir = updateConfig.getDataDir();
this.lonSize = updateConfig.getLonSize();
this.latSize = updateConfig.getLatSize();
this.levSize = updateConfig.getLevSize();
// // FIXME 2024/5/10: 维度信息取决于数据下载，目前需要两个配置一起更新
this.timeToLive = updateConfig.getTimeToLive();
JSONUtils.pojo2jsonFile(this, filePath);
log.info("配置文件 {} 更新为: {}", filePath, updateConfig);
}
public void valid() {
}
private void initLonList() {
double lonStart = dataConfig.getMinLon();
double res = dataConfig.getResolution();
lonList = new double[lonSize];
for (int i = 0; i < lonSize; i++) {
lonList[i] = lonStart + i * res;
}
}
private void initLatList() {
double latStart = dataConfig.getMinLat();
double res = dataConfig.getResolution();
latList = new double[latSize];
for (int i = 0; i < latSize; i++) {
latList[i] = latStart + i * res;
}
}
private void initLevList() {
levList = dataConfig.getPressureLevels();
}
}

15
weather-service/src/main/java/com/htfp/weather/griddata/common/TableStoreConf.java
Unescape Escape View File

@ -0,0 +1,15 @@
package com.htfp.weather.griddata.common;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration("tableStoreConf")
@ConfigurationProperties("tablestore")
public class TableStoreConf {
private String endpoint;
private String accessId;
private String accessKey;
private String instanceName;
}

44
weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java
Unescape Escape View File

@ -0,0 +1,44 @@
package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.TableStoreGridConfig;
import com.aliyun.tablestore.grid.TableStoreGrid;
import com.google.common.collect.Tables;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.common.TableStoreConf;
import org.checkerframework.checker.units.qual.A;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@DependsOn("tableStoreConf")
public abstract class BaseTableOperation{
protected TableStoreGrid tableStoreGrid;
// private String pathSeperator = "/";
@Autowired
private TableConfigBean tableConfigBean;
@Autowired
private TableStoreConf tableStoreConf;
@PostConstruct
public void init() {
TableStoreGridConfig config = new TableStoreGridConfig();
config.setTableStoreEndpoint(tableStoreConf.getEndpoint());
config.setAccessId(tableStoreConf.getAccessId());
config.setAccessKey(tableStoreConf.getAccessKey());
config.setTableStoreInstance(tableStoreConf.getInstanceName());
config.setDataTableName(tableConfigBean.getDataTableName());
config.setMetaTableName(tableConfigBean.getMetaTableName());
tableStoreGrid = new TableStoreGrid(config);
}
public void close() {
if (tableStoreGrid != null) {
tableStoreGrid.close();
}
}
}

53
weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java
Unescape Escape View File

@ -0,0 +1,53 @@
package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.model.TableOptions;
import com.alicloud.openservices.tablestore.model.search.FieldSchema;
import com.alicloud.openservices.tablestore.model.search.FieldType;
import com.alicloud.openservices.tablestore.model.search.IndexSchema;
import com.htfp.weather.griddata.common.TableConfig;
import java.util.Arrays;
/**
* @Author : shiyi
* @Date : 2024/1/15 15:10
* @Description : 创建表
*/
public class CreateTable extends BaseTableOperation {
/**
* we must create store before we can use it.
* @throws Exception
*/
private void createStore() throws Exception {
TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE, 1);
this.tableStoreGrid.createStore(tableOptions);
}
/**
* this example create an index which contains these columns: status, tag1, tag2, create_time.
* you can create an index which contains any other columns.
*
* @throws Exception
*/
private void createIndex() throws Exception {
IndexSchema indexSchema = new IndexSchema();
indexSchema.setFieldSchemas(Arrays.asList(
new FieldSchema("status", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true),
new FieldSchema("ref_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true)
));
this.tableStoreGrid.createMetaIndex(TableConfig.META_INDEX_NAME, indexSchema);
}
public static void main(String[] args) throws Exception {
CreateTable example = new CreateTable();
try {
example.createStore();
example.createIndex();
} finally {
example.close();
}
}
}

130
weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java
Unescape Escape View File

@ -0,0 +1,130 @@
package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.AsyncClient;
import com.alicloud.openservices.tablestore.model.*;
import com.aliyun.tablestore.grid.GridDataDeleter;
import com.aliyun.tablestore.grid.core.RequestBuilder;
import com.aliyun.tablestore.grid.model.DeleteDataParam;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.htfp.weather.griddata.common.TableConfig;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.Field;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/**
* @Author : shiyi
* @Date : 2024/2/2 16:53
* @Description : 批量删除数据
*/
@Slf4j
public class DataDeleter extends BaseTableOperation {
/** 删除指定datasetId的所有数据
*
* @param dataSetId
* @return
* @throws Exception
*/
public void deleteDataset(String dataSetId) throws Exception {
GridDataDeleter deleter = tableStoreGrid.getDataDeleter(tableStoreGrid.getDataSetMeta(dataSetId));
deleter.delete(dataSetId);
}
private AsyncClient getClient() throws IllegalAccessException, NoSuchFieldException {
Field declaredField = this.tableStoreGrid.getClass().getDeclaredField("asyncClient");
declaredField.setAccessible(true);
AsyncClient client = (AsyncClient) declaredField.get(this.tableStoreGrid);
return client;
}
private void batchWriteRow() throws Exception {
AsyncClient client = getClient();
BatchWriteRowRequest batchWriteRowRequest = new BatchWriteRowRequest();
//构造rowDeleteChange1。
// PrimaryKeyBuilder pk1Builder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
// pk1Builder.addPrimaryKeyColumn("_id", PrimaryKeyValue.fromString("pk"));
// //设置数据表名称。
// RowDeleteChange rowDeleteChange1 = new RowDeleteChange("<TABLE_NAME1>", pk1Builder.build());
// //添加到batch操作中。
// batchWriteRowRequest.addRowChange(rowDeleteChange1);
//构造rowDeleteChange2。
String dataSetId = "UTC-20230910";
GridDataSetMeta dataSetMeta = this.tableStoreGrid.getDataSetMeta(dataSetId);
Set<Integer> refTimes = dataSetMeta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet());
for (int t=0;t<3;t++) {
for (int z=0; z<8; z++) {
DeleteDataParam param = new DeleteDataParam(TableConfig.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z);
//添加到batch操作中。
batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange());
}
}
BatchWriteRowResponse response = client.batchWriteRow(batchWriteRowRequest, null).get();
System.out.println("是否全部成功：" + response.isAllSucceed());
if (!response.isAllSucceed()) {
for (BatchWriteRowResponse.RowResult rowResult : response.getFailedRows()) {
System.out.println("失败的行：" + batchWriteRowRequest.getRowChange(rowResult.getTableName(), rowResult.getIndex()).getPrimaryKey());
System.out.println("失败原因：" + rowResult.getError());
}
/**
* 可以通过createRequestForRetry方法再构造一个请求对失败的行进行重试。此处只给出构造重试请求的部分。
* 推荐的重试方法是使用SDK的自定义重试策略功能，支持对batch操作的部分行错误进行重试。设置重试策略后，调用接口处无需增加重试代码。
*/
BatchWriteRowRequest retryRequest = batchWriteRowRequest.createRequestForRetry(response.getFailedRows());
}
}
private void getRange(String pk, String startPkValue, String endPkValue) throws NoSuchFieldException, IllegalAccessException, ExecutionException, InterruptedException {
AsyncClient client = getClient();
//设置数据表名称。
RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfig.DATA_TABLE_NAME);
//设置起始主键。
PrimaryKeyBuilder primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
primaryKeyBuilder.addPrimaryKeyColumn(pk, PrimaryKeyValue.fromString(startPkValue));
rangeRowQueryCriteria.setInclusiveStartPrimaryKey(primaryKeyBuilder.build());
//设置结束主键。
primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder();
primaryKeyBuilder.addPrimaryKeyColumn(pk, PrimaryKeyValue.fromString(endPkValue));
rangeRowQueryCriteria.setExclusiveEndPrimaryKey(primaryKeyBuilder.build());
rangeRowQueryCriteria.setMaxVersions(1);
System.out.println("GetRange的结果为：");
while (true) {
GetRangeResponse getRangeResponse = client.getRange(new GetRangeRequest(rangeRowQueryCriteria),null).get();
for (Row row : getRangeResponse.getRows()) {
System.out.println(row);
}
//如果NextStartPrimaryKey不为null，则继续读取。
if (getRangeResponse.getNextStartPrimaryKey() != null) {
rangeRowQueryCriteria.setInclusiveStartPrimaryKey(getRangeResponse.getNextStartPrimaryKey());
} else {
break;
}
}
}
public static void main(String[] args) throws Exception {
DataDeleter dataDeleter = new DataDeleter();
// dataDeleter.batchWriteRow();
try {
dataDeleter.deleteDataset("UTC-20230910");
} finally {
dataDeleter.close();
}
// dataDeleter.getRange("_id", "UTC-20230910", "UTC-20230911");
}
}

137
weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java
Unescape Escape View File

@ -0,0 +1,137 @@
package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.GridDataFetcher;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.htfp.weather.griddata.common.TableConfigBean;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import ucar.ma2.Array;
import javax.annotation.Resource;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/22 13:53
* @Description : 根据维度信息获取数据
*/
@Slf4j @Component
@DependsOn({"tableStoreConf","tableConfigBean"})
public class GfsDataFetcher extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
/**
* 获取指定变量的四维数据(time, lev, lat, lon)
* @param dataSetId 数据id
* @param variables 变量名列表
* @param origin 维度起始索引
* @param shape 数据形状
* @return {@link GridDataSet}
* @throws Exception
*/
public GridDataSet queryByTableStore(String dataSetId, List<String> variables, int[] origin, int[] shape) throws Exception {
GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(tableStoreGrid.getDataSetMeta(dataSetId));
fetcher.setVariablesToGet(variables);
fetcher.setOriginShape(origin, shape);
GridDataSet gridDataSet = fetcher.fetch();
// Grid4D grid4D = fetcher.fetch().getVariable(variable);
return gridDataSet;
}
/**
* 获取指定变量的二维全平面数据
* @param variable 变量名
* @param iLev 高度索引
* @return {@link Array}
* @throws Exception
*/
public Array getPlane(String dataSetId, String variable, int iTime, int iLev) throws Exception {
int[] origin = new int[] {iTime, iLev, 0, 0};
int[] shape = new int[] {1, 1, tableConfigBean.latSize, tableConfigBean.lonSize};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable),
origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();
System.out.println("Shape: " + array.shapeToString());
System.out.println("Data: " + array);
return array;
}
/**
* 获取给定变量的二维平面范围数据
* @param variable 变量名
* @param iLev 高度索引
* @param iLonStart 起始经度索引
* @param iLatLength 纬度格点数量
* @param iLonLength 经度格点数量
* @return {@link Array}
* @throws Exception
*/
public Array getPlane(String dataSetId, String variable, int iTime, int iLev, int iLatStart, int iLonStart, int iLatLength, int iLonLength) throws Exception {
int[] origin = new int[] {iTime, iLev, iLatStart, iLonStart};
int[] shape = new int[] {1, 1, iLatLength, iLonLength};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable),
origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();
System.out.println("Shape: " + array.shapeToString());
System.out.println("Data: " + array);
return array;
}
/**
* 获取指定变量在某位置随高度的分布
* @param variable 变量名
* @param iLat 纬度索引
* @param iLon 经度索引
* @return {@link Array}
* @throws Exception
*/
public Array getProfile(String dataSetId, String variable, int iTime, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTime, 0, iLat, iLon};
int[] shape = new int[] {1, tableConfigBean.levSize, 1, 1};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable),
origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();
return array;
}
/**
* 获取指定变量在某经纬高处的预报序列
* @param dataSetId 数据集ID
* @param variable 变量名
* @param iLev 高度索引
* @param iLat 纬度索引
* @param iLon 经度索引
* @throws Exception
*/
public Array getSeries(String dataSetId, String variable, int[] iTimeList, int iLev, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTimeList[0], iLev, iLat, iLon};
int[] shape = new int[] {iTimeList.length, 1, 1, 1};
GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), origin, shape);
Array array = gridDataSet.getVariable(variable).toArray();
System.out.println("Shape: " + array.shapeToString());
System.out.println("Data: " + array);
return array;
}
/**
* 获取指定变量在某位置随时间的变化
* @param dataSetId 数据集ID
* @param variableList 变量名列表
* @param iTImes 时间索引数组
* @param iLev 高度索引
* @param iLat 纬度索引
* @param iLon 经度索引
* @return
* @throws Exception
*/
public GridDataSet getSeries(String dataSetId, List<String> variableList, int[] iTImes, int iLev, int iLat, int iLon) throws Exception {
int[] origin = new int[] {iTImes[0], iLev, iLat, iLon};
int[] shape = new int[] {iTImes.length, 1, 1, 1};
return queryByTableStore(dataSetId, variableList, origin, shape);
}
}

264
weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java
Unescape Escape View File

@ -0,0 +1,264 @@
package com.htfp.weather.griddata.operation;
import com.aliyun.tablestore.grid.GridDataWriter;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.MeteoUtils;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.sun.xml.internal.bind.v2.TODO;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import ucar.ma2.*;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;
import ucar.nc2.Variable;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import java.io.File;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* @author shi_y
*/
@Slf4j
@Component
@DependsOn("tableConfigBean")
public class GfsDataImport extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
/**
* init meta data to table store.
*
* @param dataSetID
* @param dataType
* @param variables
* @param shape // * @param attribute meta表的属性列
* @return
* @throws Exception
*/
private GridDataSetMeta initMeta(String dataSetID, DataType dataType, List<String> variables, int[] shape) throws Exception {
GridDataSetMeta meta = new GridDataSetMeta(
dataSetID,
dataType,
variables,
shape[0],
shape[1],
shape[2],
shape[3],
new StoreOptions(StoreOptions.StoreType.SLICE));
// for (Map.Entry<String, Object> entry : attributes.entrySet()) {
// meta.addAttribute(entry.getKey(), entry.getValue());
// }
meta.addAttribute("status", "INIT");
meta.addAttribute("create_time", System.currentTimeMillis());
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
/**
* update meta and set status to DONE when data import finished.
*
* @param meta
* @return
* @throws Exception
*/
public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception {
meta.addAttribute("status", "DONE");
tableStoreGrid.updateDataSetMeta(meta);
return meta;
}
public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception {
meta.addAttribute("status", "DONE");
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
public void importData(OffsetDateTime refTime) throws Exception {
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES);
}
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfigBean.variableList);
int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 记录到数据库属性中
// TODO 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长
List<Future<ImportResult>> futures = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
String file = fileList.get(i);
int iTime = i;
futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime)));
}
for (Future<ImportResult> future : futures) {
ImportResult importResult = future.get();
forecastHours.add(String.valueOf(importResult.getForcastHour()));
}
log.info("[tablestore] 数据导入完成, forecastHours: {}", forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
}
/**
* read data from netcdf file and write data to table store.
*
* @param meta 数据元信息
* @param file netcdf文件路径
* @param iTime 文件序号
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) {
log.info("[tablestore] 导入文件数据开始,_t={}: {}", iTime, file);
int time = 0;
try {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
NetcdfFile ncFile = NetcdfFiles.open(file);
// 相对reftime的小时数，文件可能缺失，因此time可能是不连续的，但是iTime是连续的
time = (int) ncFile.findVariable("time").read().getDouble(0);
for (String variableName : meta.getVariables()) {
if ("Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)) {
continue;
}
Variable variable = ncFile.findVariable(variableName);
if (variable != null) {
for (int z = 0; z < meta.getzSize(); z++) {
// 注意高度索引是递增排序，即z=0对应高层低气压值
Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()});
writer.writeGrid2D(variable.getShortName(), iTime, z, grid2D);
}
} else {
log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName);
}
}
// 导入风速风向
importWind(meta, ncFile, iTime);
ncFile.close();
} catch (Exception e) {
log.error("[tablestore] 导入文件数据失败,_t={}: {}", iTime, file, e);
return new ImportResult(false, file, time, iTime);
}
log.info("[tablestore] 导入文件数据成功,_t={}: {}", iTime, file);
return new ImportResult(true, file, time, iTime);
}
private void transferUnit(String variableName, Array array) {
if (GfsVariableNameEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
}
}
private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception {
// TODO 2024/5/8: 风速风向需要保存到文件中
Variable uwnd = ncFile.findVariable("u-component_of_wind_isobaric");
Variable vwnd = ncFile.findVariable("v-component_of_wind_isobaric");
int xsize = meta.getxSize();
int ysize = meta.getySize();
int zsize = meta.getzSize();
if (uwnd == null || vwnd == null) {
return;
}
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
for (int z = 0; z < zsize; z++) {
Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize});
Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray);
Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray);
Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_speed_isobaric", iTime, z, speedGrid2D);
Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(),
new int[]{0, 0}, new int[]{xsize, ysize});
writer.writeGrid2D("Wind_direction_isobaric", iTime, z, dirGrid2D);
}
}
private List<String> getFileVariables(List<String> variables) {
List<String> fileVariables = new ArrayList<>();
for (String variable : variables) {
String fileVariableName = GfsVariableNameEnum.getGfsVariableName(variable);
if (fileVariableName != null) {
fileVariables.add(fileVariableName);
} else {
log.warn("GFS数据文件中没有 {} 对应的变量", variable);
}
}
return fileVariables;
}
/**
* 获取文件目录中的所有文件绝对路径
*
* @param refTime，UTC起报时间
* @return
*/
private List<String> getFiles(OffsetDateTime refTime) {
String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File fileDir = new File(tableConfigBean.dataDir + dataFolder);
if (!fileDir.exists()) {
log.warn("文件夹 {} 不存在", fileDir);
return null;
}
List<String> fileList = new ArrayList<>();
String[] files = fileDir.list((dir, name) -> {
return name.startsWith("BJT") && name.endsWith("grib2"); // 判断文件名是否以“BJT”开头
});
if (files == null || files.length == 0) {
log.warn("文件夹 {} 中没有符合条件的文件", fileDir);
return fileList;
}
for (String filename : files) {
fileList.add(fileDir.getAbsolutePath() + File.separator + filename);
}
return fileList;
}
@PreDestroy
public void end() {
close();
}
@Data
public static class ImportResult {
private boolean success;
private String file;
private int forcastHour;
private int iTime;
public ImportResult(boolean success, String file, int forcastHour, int iTime) {
this.success = success;
this.file = file;
this.forcastHour = forcastHour;
this.iTime = iTime;
}
}
}

9
weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java
Unescape Escape View File

@ -0,0 +1,9 @@
package com.htfp.weather.griddata.operation;
/**
* @Author : shiyi
* @Date : 2024/2/2 13:47
* @Description : 获取数据
*/
public interface IDataFetch {
}

51
weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java
Unescape Escape View File

@ -0,0 +1,51 @@
package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.model.search.sort.FieldSort;
import com.alicloud.openservices.tablestore.model.search.sort.Sort;
import com.alicloud.openservices.tablestore.model.search.sort.SortOrder;
import com.aliyun.tablestore.grid.core.QueryBuilder;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.QueryGridDataSetResult;
import com.aliyun.tablestore.grid.model.QueryParams;
import com.htfp.weather.griddata.common.TableConfigBean;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.Resource;
import java.util.Collections;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/3/4 17:45
* @Description : 查询属性表
*/
@Component
public class QueryMeta extends BaseTableOperation {
@Resource
TableConfigBean tableConfigBean;
/**
* 查询最新的数据集信息
*
* @return
* @throws Exception
*/
public GridDataSetMeta getLastGridDataSetMeta() throws Exception {
QueryGridDataSetResult result = tableStoreGrid.queryDataSets(
tableConfigBean.metaIndexName,
QueryBuilder.and()
.equal("status", "DONE")
.build(),
new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("reference_time", SortOrder.DESC)))));
List<GridDataSetMeta> gridDataSetMetas = result.getGridDataSetMetas();
if (CollectionUtils.isEmpty(gridDataSetMetas)) {
throw new RuntimeException("meta table为空");
}
return gridDataSetMetas.get(0);
}
public String queryLastReferenceTime() throws Exception {
GridDataSetMeta gridDataSetMeta = getLastGridDataSetMeta();
return gridDataSetMeta.getGridDataSetId();
}
}

29
weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java
Unescape Escape View File

@ -0,0 +1,29 @@
package com.htfp.weather.griddata.operation;
import com.alicloud.openservices.tablestore.model.TableOptions;
import com.htfp.weather.griddata.common.TableConfig;
/**
* @Author : shiyi
* @Date : 2024/2/3 17:54
* @Description :
*/
public class UpdateTable extends BaseTableOperation {
private void update() throws Exception {
TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE);
this.tableStoreGrid.updateStoreOption(tableOptions);
}
public static void main(String[] args) {
TableConfig.initDatasetConfig();
UpdateTable example = new UpdateTable();
try {
example.update();
} catch (Exception e) {
e.printStackTrace();
} finally {
example.close();
}
}
}

45
weather-service/src/main/java/com/htfp/weather/griddata/utils/CoordinateUtils.java
Unescape Escape View File

@ -0,0 +1,45 @@
package com.htfp.weather.griddata.utils;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* @Author : shiyi
* @Date : 2024/2/28 13:33
* @Description : 物理坐标索引
*/
public class CoordinateUtils {
public static final String DATE_TIME_PATTERN = "yyyy-MM-dd'T'HH:mmxxx";
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(DATE_TIME_PATTERN);
public int getLevelIndex(int lev) {
return 0;
}
public int getLatitudeIndex(double lat) {
return 0;
}
public int getLongitudeIndex(double lon) {
return 0;
}
public int getTimeIndex(long time) {
return 0;
}
/**
* 获取当前时刻对应数据库的起报时间（datasetId）
* @param timeStr
* @return
*/
public String getRefTimeInDataBase(String timeStr) {
OffsetDateTime utcTime = OffsetDateTime.parse(timeStr, DATE_TIME_FORMATTER)
.withOffsetSameInstant(ZoneOffset.ofHours(0));
// 获取utcTime这一天对应的00
return utcTime.withHour(2).format(DATE_TIME_FORMATTER);
}
}

39
weather-service/src/main/java/com/htfp/weather/griddata/utils/GfsUtils.java
Unescape Escape View File

@ -0,0 +1,39 @@
package com.htfp.weather.griddata.utils;
import org.apache.commons.lang3.StringUtils;
import ucar.nc2.Attribute;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;
import java.io.IOException;
import java.time.ZonedDateTime;
/**
* @Author : shiyi
* @Date : 2024/3/4 10:19
* @Description : Gfs数据工具类
*/
public class GfsUtils {
/**
* 从cdf文件中获取起报时间，并转换为时区字符串格式
* @param ncFile
* @return 时区字符串："yyyy-MM-dd'T'HH:mmxxx"，例如"2023-09-05T08:00+08:00"
* @throws IOException
*/
public static String getRefTime(NetcdfFile ncFile) throws IOException {
Variable time = ncFile.findVariable("reftime");
if ( time!= null) {
int hour = (int) time.read().getDouble(0);
Attribute units = time.findAttribute("units");
if (units != null) {
String stringValue = units.getStringValue();
if (StringUtils.isNotBlank(stringValue)) {
// "Hour since 2023-09-05T00:00:00Z" // utc时间
ZonedDateTime zonedDateTime = ZonedDateTime.parse(stringValue.substring(11)).plusHours(hour);
return zonedDateTime.format(CoordinateUtils.DATE_TIME_FORMATTER);
}
}
}
return null;
}
}

81
weather-service/src/main/java/com/htfp/weather/info/CaiYunInfo.java
Unescape Escape View File

@ -0,0 +1,81 @@
package com.htfp.weather.info;
import java.util.HashMap;
import java.util.Map;
/**
* @Author : shiyi
* @Date : 2024/4/18 13:37
* @Description : 彩云天气相关映射
*/
public class CaiYunInfo {
private static final Map<String, String> skyconMap = new HashMap<>();
private static final Map<String, String> alertNameMap = new HashMap<>();
private static final Map<String, String> alertColorMap = new HashMap<>();
static {
// 彩云天气天气现象映射
skyconMap.put("CLEAR_DAY", "晴（白天）");
skyconMap.put("CLEAR_NIGHT", "晴（夜间）");
skyconMap.put("PARTLY_CLOUDY_DAY", "多云（白天）");
skyconMap.put("PARTLY_CLOUDY_NIGHT", "多云（夜间）");
skyconMap.put("CLOUDY", "阴");
skyconMap.put("LIGHT_HAZE", "轻度雾霾");
skyconMap.put("MODERATE_HAZE", "中度雾霾");
skyconMap.put("HEAVY_HAZE", "重度雾霾");
skyconMap.put("LIGHT_RAIN", "小雨");
skyconMap.put("MODERATE_RAIN", "中雨");
skyconMap.put("HEAVY_RAIN", "大雨");
skyconMap.put("STORM_RAIN", "暴雨");
skyconMap.put("FOG", "雾");
skyconMap.put("LIGHT_SNOW", "小雪");
skyconMap.put("MODERATE_SNOW", "中雪");
skyconMap.put("HEAVY_SNOW", "大雪");
skyconMap.put("STORM_SNOW", "暴雪");
skyconMap.put("DUST", "浮尘");
skyconMap.put("SAND", "沙尘");
skyconMap.put("WIND", "大风");
skyconMap.put("THUNDER_SHOWER", "雷阵雨");
skyconMap.put("HAIL", "冰雹");
skyconMap.put("SLEET", "雨夹雪");
// 彩云天气预警类型映射
alertNameMap.put("01", "台风");
alertNameMap.put("02", "暴雨");
alertNameMap.put("03", "暴雪");
alertNameMap.put("04", "寒潮");
alertNameMap.put("05", "大风");
alertNameMap.put("06", "沙尘暴");
alertNameMap.put("07", "高温");
alertNameMap.put("08", "干旱");
alertNameMap.put("09", "雷电");
alertNameMap.put("10", "冰雹");
alertNameMap.put("11", "霜冻");
alertNameMap.put("12", "大雾");
alertNameMap.put("13", "霾");
alertNameMap.put("14", "道路结冰");
alertNameMap.put("15", "森林火险");
alertNameMap.put("16", "雷雨大风");
alertNameMap.put("17", "春季沙尘天气趋势预警");
alertNameMap.put("18", "沙尘");
//
alertColorMap.put("00", "白色");
alertColorMap.put("01", "蓝色");
alertColorMap.put("02", "黄色");
alertColorMap.put("03", "橙色");
alertColorMap.put("04", "红色");
}
public static String getSkyConName(String skycon) {
return skyconMap.getOrDefault(skycon, "未知天气现象");
}
public static String getAlertName(String code) {
return alertNameMap.getOrDefault(code.substring(0,2), "未知预警类型");
}
public static String getAlertColor(String code) {
return alertColorMap.getOrDefault(code.substring(2,4), "未知预警等级");
}
}

14
weather-service/src/main/java/com/htfp/weather/info/Constant.java
Unescape Escape View File

@ -0,0 +1,14 @@
package com.htfp.weather.info;
/**
* @Author : shiyi
* @Date : 2024/5/8 12:02
* @Description :
*/
public class Constant {
public static final String DATA_SET_ID_FORMAT_STRING = "'UTC-'yyyyMMdd.HH";
public static final String DATA_FOLDER_STRING = "'UTC-'yyyyMMdd.HH";
public static final String UTC_TIME_STRING = "'UTC-'yyyyMMdd.HH";
public static final String BJT_TIME_STRING = "'BJT-'yyyyMMdd.HH";
public static final String API_TIME_STRING = "yyyy-MM-dd'T'HH:mmxxx";
}

42
weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java
Unescape Escape View File

@ -0,0 +1,42 @@
package com.htfp.weather.info;
/**
* @Author : shiyi
* @Date : 2024/4/28 16:31
* @Description : GFS变量枚举
*/
public enum GfsDownloadVariableEnum {
//
TEMPERATURE("TMP", "温度"),
U_WIND("UGRD", "东西风分量"),
V_WIND("VGRD", "南北风分量"),
CLOUD("TCDC", "云量"),
RELATIVE_HUMIDITY("RH", "相对湿度"),
VERTICAL_VELOCITY("DZDT", "垂直速度"),
;
private final String code;
private final String info;
GfsDownloadVariableEnum(String code, String info) {
this.code = code;
this.info = info;
}
public String getCode() {
return code;
}
public String getInfo() {
return info;
}
public static boolean contains(String test) {
for (GfsDownloadVariableEnum c : GfsDownloadVariableEnum.values()) {
if (c.getCode().equals(test)) {
return true;
}
}
return false;
}
}

54
weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java
Unescape Escape View File

@ -0,0 +1,54 @@
package com.htfp.weather.info;
/**
* @Author : shiyi
* @Date : 2024/4/28 16:45
* @Description : 高度层枚举
*/
public enum GfsLevelsEnum {
//
SURFACE(9999, "地面"),
PRES_1000hPa(1000, "1000 hPa"),
PRES_975hPa(975, "975 hPa"),
PRES_950hPa(950, "950 hPa"),
PRES_925hPa(925, "925 hPa"),
PRES_900hPa(900, "900 hPa"),
PRES_850hPa(850, "850 hPa"),
PRES_800hPa(800, "800 hPa"),
PRES_750hPa(750, "750 hPa"),
PRES_700hPa(700, "700 hPa"),
PRES_650hPa(650, "650 hPa"),
PRES_600hPa(600, "600 hPa"),
PRES_550hPa(550, "550 hPa"),
PRES_500hPa(500, "500 hPa"),
PRES_450hPa(450, "450 hPa"),
PRES_400hPa(400, "400 hPa"),
PRES_350hPa(350, "350 hPa"),
;
private final Integer code;
private final String info;
GfsLevelsEnum(Integer code, String info) {
this.code = code;
this.info = info;
}
public Integer getCode() {
return code;
}
public String getInfo() {
return info;
}
public static boolean contains(Integer test) {
for (GfsLevelsEnum c : GfsLevelsEnum.values()) {
if (c.getCode().equals(test)) {
return true;
}
}
return false;
}
}

76
weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
Unescape Escape View File

@ -0,0 +1,76 @@
package com.htfp.weather.schedule;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.info.Constant;
import com.htfp.weather.web.exception.AppExcpetion;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.checkerframework.checker.units.qual.A;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/5/8 11:36
* @Description : 更新数据库数据，包括下载、导入、清楚本地过期数据
*/
@Component
@Slf4j
@DependsOn({"gfsDataImport", "gfsDownloader"})
public class GridDataProcesser {
@Resource
GfsDataImport gfsDataImport;
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataConfig gfsDataConfig;
public void dailyDataProcess() throws Exception {
gfsDownloader.iniTimeSetting();
OffsetDateTime refTime = gfsDownloader.getRefTime();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
boolean allComplete = gfsDownloader.downloadAll(fileInfoList);
if (allComplete) {
try {
gfsDataImport.importData(refTime);
} catch (AppExcpetion e) {
log.info("导入数据失败");
}
}
}
public void clearExpiredData() {
// String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING));
File dataDir = new File(gfsDataConfig.getSaveRoot());
if (!dataDir.exists()) {
log.warn("文件夹 {} 不存在", dataDir);
return;
}
OffsetDateTime now = OffsetDateTime.now();
for (File subDir : dataDir.listFiles()) {
OffsetDateTime refTime = LocalDateTime.parse(subDir.getName(), DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)).atOffset(ZoneOffset.UTC);
if (refTime.isBefore(now.minusDays(3))) {
try {
FileUtils.deleteDirectory(subDir);
log.info("删除过期数据文件夹成功： {}", subDir.getName());
} catch (IOException e) {
log.info("删除过期数据文件夹失败： {}", subDir.getName());
}
}
}
}
}

135
weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java
Unescape Escape View File

@ -0,0 +1,135 @@
package com.htfp.weather.utils;
import java.time.*;
import java.time.format.DateTimeFormatter;
/**
* @Author : shiyi
* @Date : 2024/1/23 18:18
* @Description : 日期时间工具类
*/
public class DateTimeUtils {
public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mmxxx";
public static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ofPattern(DEFAULT_PATTERN);
/**
* 将某时区的时间字符串更改为其他时区的时间字符串,
*
* @param dateTimeStr 输入的时间字符串，需包含时区信息，如+08:00
* @param offsetTo 输出时间的时区，如8代表东八区
* @return 更改时区之后的时间字符串，输出格式与输入格式一致
*/
public static String convertZonedDateTime(String dateTimeStr, int offsetTo, String formatStr) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr);
// 转换为UTC时区的ZonedDateTime对象，时间点保持一致
OffsetDateTime offsetDateTime = OffsetDateTime.parse(dateTimeStr, formatter)
.withOffsetSameInstant(ZoneOffset.ofHours(offsetTo));
return offsetDateTime.format(formatter);
}
/**
* 以指定的格式，将某时区的时间字符串更改为其他时区的时间字符串
*
* @param dateTimeStr 输入的时间字符串，不需要包含时区信息，如果含有时区信息将会被忽略
* @param zoneFrom 输入时间的时区, dateTimeStr中的时区信息将被忽略
* @param zoneTo 输出时间的时区，如8代表东八区
* @param formatStr 输入和输出的时间字符串格式，可以不包含时区信息
* @return 指定时区表示的时间字符串
*/
public static String convertZonedDateTime(String dateTimeStr, int zoneFrom, int zoneTo, String formatStr) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr);
ZoneId zoneIdFrom = ZoneId.ofOffset("", ZoneOffset.ofHours(zoneFrom));
ZoneId zoneIdTo = ZoneId.ofOffset("", ZoneOffset.ofHours(zoneTo));
// LocalDateTime解析将会忽略dateTimeStr中的时区信息
// OffsetDateTime.parse(dateTimeStr)
ZonedDateTime zonedDateTime = LocalDateTime.parse(dateTimeStr, formatter)
.atZone(zoneIdFrom)
.withZoneSameInstant(zoneIdTo);
return zonedDateTime.format(formatter);
}
public static OffsetDateTime getUTCDateTime(OffsetDateTime localDateTime) {
return localDateTime.withOffsetSameInstant(ZoneOffset.UTC);
}
// public static LocalDateTime getUTCDateTime(String timeString, String formatStr) {
// DateTimeFormatter formatter = new DateTimeFormatterBuilder()
// .appendPattern(formatStr)
// .toFormatter();
//
// // 转换为UTC时区的ZonedDateTime对象，时间点保持一致
// ZonedDateTime utcZonedDateTime = LocalDateTime.parse(timeString, formatter)
// .atZone(ZoneId.systemDefault())
// .withZoneSameInstant(ZoneId.of("UTC"));
// return utcZonedDateTime.toLocalDateTime();
// }
public static ZonedDateTime getLocalZoneDateTime(OffsetDateTime utcDateTime) {
ZonedDateTime zonedDateTime = utcDateTime.toZonedDateTime().withZoneSameInstant(ZoneId.systemDefault());
return zonedDateTime;
}
/**
* 将UTC时间字符串转换为将北京时时间字符串
*
* @param utcTimeStr UTC时间字符串
* @param formatStr 输入和输出的字符串格式
* @return 北京时（东八区）时间字符串
*/
public static String getBJTDateTimeStringFromUTC(String utcTimeStr, String formatStr) {
return convertZonedDateTime(utcTimeStr, 0, 8, formatStr);
}
/**
* 将北京时时间字符串转换为UTC时间字符串
*
* @param bjtTimeStr 北京时（东八区）时间字符串
* @param formatStr 输入和输出的字符串格式
* @return UTC时间字符串
*/
public static String getUTCDateTimeStringFromBJT(String bjtTimeStr, String formatStr) {
return convertZonedDateTime(bjtTimeStr, 8, 0, formatStr);
}
/** 获取指定日期的时间戳（毫秒）
* @param dateTimeStr 时区时间字符串
* @param formatStr 指定日期字符串的格式
* @return 毫秒时间戳
*/
public static long getTimestampMilli(String dateTimeStr, String formatStr) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr);
return OffsetDateTime.parse(dateTimeStr, formatter).toInstant().toEpochMilli();
}
public static long getTimestampMilli(String dateTimeStr) {
return OffsetDateTime.parse(dateTimeStr, DEFAULT_FORMATTER).toInstant().toEpochMilli();
}
/**
* 将时间戳转换为时间字符串
* @param timestampMilli
* @param zoneOffset
* @return yyyy-MM-dd'T'HH:mmxxx 格式字符串
*/
public static String getTimeStringFromMilli(long timestampMilli, int zoneOffset) {
Instant instant = Instant.ofEpochMilli(timestampMilli);
// 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间
ZonedDateTime zonedDateTime = instant.atZone(ZoneId.ofOffset("", ZoneOffset.ofHours(zoneOffset)));
return zonedDateTime.format(DEFAULT_FORMATTER);
}
public static String getTimeStringFromMilli(long timestampMilli, String zoneId) {
Instant instant = Instant.ofEpochMilli(timestampMilli);
// 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间
ZonedDateTime zonedDateTime = instant.atZone(ZoneId.of(zoneId));
return zonedDateTime.format(DEFAULT_FORMATTER);
}
/**
* 将时间戳转换为本地时区的字符串
* @param timestampMilli
* @return yyyy-MM-dd'T'HH:mmxxx 格式字符串
*/
public static String getTimeStringFromMilli(long timestampMilli) {
Instant instant = Instant.ofEpochMilli(timestampMilli);
// 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间
ZonedDateTime zonedDateTime = instant.atZone(ZoneId.systemDefault());
return zonedDateTime.format(DEFAULT_FORMATTER);
}
}

9
weather-service/src/main/java/com/htfp/weather/utils/DownloadUtils.java
Unescape Escape View File

@ -0,0 +1,9 @@
package com.htfp.weather.utils;
/**
* @Author : shiyi
* @Date : 2024/4/25 18:31
* @Description : 文件下载https://www.jianshu.com/p/3b269082cbbb
*/
public class DownloadUtils {
}

86
weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java
Unescape Escape View File

@ -0,0 +1,86 @@
package com.htfp.weather.utils;
import lombok.extern.slf4j.Slf4j;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* @Author : shiyi
* @Date : 2024/1/23 16:19
* @Description : HttpClient工具类
*/
@Slf4j
public class HttpClientUtils {
private static final OkHttpClient OKHTTP_CLIENT = new OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.writeTimeout(60, TimeUnit.SECONDS)
.build();
// public static <T> T sendGet(String url, Map<String, String> params, Class<T> responseClass) {
// MultiValueMap<String, String> queryParams = new LinkedMultiValueMap<>();
// if (params != null) {
// params.forEach(queryParams::add);
// }
// Mono<T> mono = WebClient.create(url).method(HttpMethod.GET)
// .uri(uriBuilder -> uriBuilder
// .queryParams(queryParams)
// .build()).retrieve().bodyToMono(responseClass);
// T responseBody;
// try {
// responseBody = mono.block();
// } catch (WebClientResponseException e) {
// log.error("Get 请求错误, {}", e.getMessage());
// return null;
// }
// return responseBody;
// }
public static <T> T sendGet(String url, Map<String, String> params, Class<T> responseClass) throws Exception {
String jsonStr = sendGet(url, params);
return JSONUtils.json2pojo(jsonStr, responseClass);
}
public static String sendGet(String url, Map<String, String> params) throws IOException {
StringBuilder stringBuilder = new StringBuilder();
if (!CollectionUtils.isEmpty(params)) {
params.keySet().forEach(res -> {
if (StringUtils.isNotBlank(stringBuilder)) {
stringBuilder.append("&");
} else {
stringBuilder.append("?");
}
try {
stringBuilder.append(String.format("%s=%s", res, URLEncoder.encode(params.get(res), "UTF-8")));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
});
}
Request request = new Builder()
.url(url + stringBuilder)
.build();
try (Response response = OKHTTP_CLIENT.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("okhttp Unexpected code " + response);
}
ResponseBody body = response.body();
if (body != null) {
return body.string();
} else {
return null;
}
}
}
}

92
weather-service/src/main/java/com/htfp/weather/utils/JSONUtils.java
Unescape Escape View File

@ -0,0 +1,92 @@
package com.htfp.weather.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class JSONUtils {
private final static ObjectMapper objectMapper = new ObjectMapper();
private JSONUtils() {
}
public static ObjectMapper getInstance() {
return objectMapper;
}
/**
* javaBean,list,array convert to json string
*/
public static String obj2json(Object obj) throws Exception {
return objectMapper.writeValueAsString(obj);
}
/**
* json string convert to javaBean
*/
public static <T> T json2pojo(String jsonStr, Class<T> clazz)
throws Exception {
return objectMapper.readValue(jsonStr, clazz);
}
/**
* json string convert to map
*/
public static <T> Map<String, Object> json2map(String jsonStr)
throws Exception {
return objectMapper.readValue(jsonStr, Map.class);
}
/**
* json string convert to map with javaBean
*/
public static <T> Map<String, T> json2map(String jsonStr, Class<T> clazz)
throws Exception {
Map<String, Map<String, Object>> map = (Map<String, Map<String, Object>>) objectMapper.readValue(jsonStr,
new TypeReference<Map<String, T>>() {
});
Map<String, T> result = new HashMap<String, T>();
for (Map.Entry<String, Map<String, Object>> entry : map.entrySet()) {
result.put(entry.getKey(), map2pojo(entry.getValue(), clazz));
}
return result;
}
/**
* json array string convert to list with javaBean
*/
public static <T> List<T> json2list(String jsonArrayStr, Class<T> clazz)
throws Exception {
List<Map<String, Object>> list = (List<Map<String, Object>>) objectMapper.readValue(jsonArrayStr,
new TypeReference<List<T>>() {
});
List<T> result = new ArrayList<T>();
for (Map<String, Object> map : list) {
result.add(map2pojo(map, clazz));
}
return result;
}
/**
* map convert to javaBean
*/
public static <T> T map2pojo(Map map, Class<T> clazz) {
return objectMapper.convertValue(map, clazz);
}
public static void pojo2jsonFile(Object obj, String filePath) throws IOException {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
objectMapper.writeValue(new File(filePath), obj);
}
}

144
weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java
Unescape Escape View File

@ -0,0 +1,144 @@
package com.htfp.weather.utils;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.DataType;
import ai.djl.ndarray.types.Shape;
import org.checkerframework.checker.units.qual.A;
import ucar.ma2.Array;
import ucar.ma2.ArrayFloat;
import ucar.ma2.Index;
import ucar.nc2.units.DateType;
import java.nio.ByteBuffer;
import java.text.DecimalFormat;
import java.util.Arrays;
/**
* @Author : shiyi
* @Date : 2024/1/24 11:32
* @Description : 气象数据处理工具类
*/
public class MeteoUtils {
/**
* 风速单位转换
*
* @param mps 风速 (m/s)
* @return 风速 (km/h)
*/
public static double mPerSecond2kmPerHour(double mps) {
return mps * 3.6;
}
/**
* @param kph 风速 (km/h)
* @return 风速 (m/s)
*/
public static double kmPerHour2mPerSecond(double kph) {
return kph / 3.6;
}
// public static double pressure2Height(double pressure) {
//
// }
//
// public static double height2Pressure(double pressure) {
//
// }
/**
* 开尔文转摄氏度
*/
public static Array kelvin2Celsius(Array array) {
for (int i = 0; i < array.getSize(); i++) {
array.setFloat(i, array.getFloat(i) - 273.15f);
}
return array;
}
public static double calculateWindSpeed(double u, double v) {
return Math.sqrt(u * u + v * v);
}
//
// @Deprecated
// public static Array calculateWindSpeed(Array u, Array v) {
// // // FIXME 2024/5/13: io耗时太多，对于一般大小的矩阵不使用
// uvValid(u, v);
// try (NDManager manager = NDManager.newBaseManager()) {
// int[] shapeArray = u.getShape();
// Shape shape = new Shape(Arrays.stream(shapeArray).mapToLong(i -> (long) i).toArray());
// NDArray uwnd = manager.create((float[]) u.getStorage(), shape);
// NDArray vwnd = manager.create((float[]) v.getStorage(), shape);
// NDArray windSpeed = (uwnd.square().add(vwnd.square())).sqrt();
// ByteBuffer byteBuffer = windSpeed.toByteBuffer();
// return Array.factory(u.getDataType(), shapeArray, byteBuffer);
// } catch (Exception e) {
// throw e;
// }
// }
private static void uvValid(Array u, Array v) {
if (u.getSize() != v.getSize()) {
throw new IllegalArgumentException("u 和 v 数据量不一致");
}
if (u.getSize() == 0 || v.getSize() == 0) {
throw new IllegalArgumentException("u 和 v 数据不能为空");
}
if (!Arrays.equals(u.getShape(), v.getShape())) {
throw new IllegalArgumentException("u 和 v 数据形状不一致");
}
if (!u.getDataType().equals(v.getDataType())) {
throw new IllegalArgumentException("u 和 v 数据类型不一致");
}
}
public static Array calculateWindSpeed(Array u, Array v) {
uvValid(u, v);
Array windSpeed = new ArrayFloat(u.getShape());
for (int i = 0; i < windSpeed.getSize(); i++) {
windSpeed.setFloat(i, (float) calculateWindSpeed(u.getFloat(i), v.getFloat(i)));
}
return windSpeed;
}
public static Array calculateWindDirection(Array u, Array v) {
uvValid(u, v);
Array windDirection = new ArrayFloat(u.getShape());
for (int i = 0; i < windDirection.getSize(); i++) {
windDirection.setFloat(i, (float) calculateWindDirection(u.getFloat(i), v.getFloat(i)));
}
return windDirection;
}
public static double calculateWindDirection(double u, double v) {
// DecimalFormat df = new DecimalFormat("#.00");
double windDirDeg;
if (nearToZero(u) && v > 0.) {
windDirDeg = 180;
} else if (nearToZero(u) && v < 0.) {
windDirDeg = 0;
} else if (u > 0. && nearToZero(v)) {
windDirDeg = 270;
} else if (u < 0. && nearToZero(v)) {
windDirDeg = 90;
} else if (nearToZero(u) && nearToZero(v)) {
windDirDeg = 0;
} else {
windDirDeg = calculateWindDirection0(u, v);
}
return (float) windDirDeg;
}
public static double calculateWindDirection0(double u, double v) {
// DecimalFormat df = new DecimalFormat("#.00");
double windDirRad = Math.atan2(v, u);
return (270 - windDirRad * 180 / Math.PI) % 360;
}
private static boolean nearToZero(double value) {
return Math.abs(value) < 1e-5;
}
}

42
weather-service/src/main/java/com/htfp/weather/utils/NdArrayUtils.java
Unescape Escape View File

@ -0,0 +1,42 @@
package com.htfp.weather.utils;
import java.util.Arrays;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/22 20:05
* @Description : 多维数组工具类
*/
public class NdArrayUtils {
/**
* 查找距离目标值最近的索引
*
* @param list
* @param target
* @return
*/
public static int findNearestIndex(double[] list, double target) {
int size = list.length;
if (size == 0) {
return -1;
}
if (size == 1) {
return 0;
}
int index = Arrays.binarySearch(list, target);
if (index >= 0) {
return index;
}
// 插入点，即最大的小于等于目标值的元素索引
int insertPoint = -(index + 1);
if (insertPoint > 1 && Math.abs(list[insertPoint - 1] - target) <= Math.abs(list[insertPoint] - target)) {
return insertPoint - 1;
}
return insertPoint;
}
}

46
weather-service/src/main/java/com/htfp/weather/utils/SpringUtil.java
Unescape Escape View File

@ -0,0 +1,46 @@
package com.htfp.weather.utils;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
/**
* @Author : shiyi
* @Date : 2024/1/5 15:51
* @Description : Spring工具类
*/
@Component
public class SpringUtil<T> implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
if(SpringUtil.applicationContext == null) {
SpringUtil.applicationContext = applicationContext;
}
}
//获取applicationContext
public static ApplicationContext getApplicationContext() {
return applicationContext;
}
//通过name获取 Bean
public static Object getBean(String name){
return getApplicationContext().getBean(name);
}
//通过class获取Bean.
public static <T> T getBean(Class<T> clazz){
return getApplicationContext().getBean(clazz);
}
//通过name,以及Clazz返回指定的Bean
public static <T> T getBean(String name,Class<T> clazz){
return getApplicationContext().getBean(name, clazz);
}
}

23
weather-service/src/main/java/com/htfp/weather/web/config/CORSFilter.java
Unescape Escape View File

@ -0,0 +1,23 @@
package com.htfp.weather.web.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
@Configuration
public class CORSFilter {
@Bean
public CorsFilter corsFilter(){
CorsConfiguration corsConfiguration = new CorsConfiguration();
corsConfiguration.addAllowedHeader("*");
corsConfiguration.addAllowedOrigin("*");
corsConfiguration.addAllowedMethod("*");
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
source.registerCorsConfiguration("/**",corsConfiguration);
return new CorsFilter(source);
}
}

113
weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
Unescape Escape View File

@ -0,0 +1,113 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.common.TableConfigBean;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.Result;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
/**
* @Author : shiyi
* @Date : 2024/4/21 15:56
* @Description : 配置文件管理
*/
@CrossOrigin
@RestController
@RequestMapping("/htfp/weather/config")
public class ConfigController {
@Resource
TableConfigBean tableConfigBean;
@Resource
GfsDataConfig gfsDataConfig;
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataImport gfsDataImport;
private final String SECRET = "htfpweather";
private void validSecret(String secret) {
if (!SECRET.equals(secret)) {
throw new AppExcpetion(ErrorCode.SECRET_ERROR);
}
}
@RequestMapping("/queryDatabaseConfig")
public Result queryDatabaseConfig() {
return Result.success(tableConfigBean);
}
@RequestMapping("/updateDatabaseConfig")
public Result updateDatabaseConfig(@RequestParam String secret, @Validated @RequestBody TableConfigBean updateConfig) {
validSecret(secret);
if (updateConfig == null) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空");
}
updateConfig.valid();
try {
tableConfigBean.writeConfig(updateConfig);
} catch (IOException e) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e);
}
return Result.success();
}
@RequestMapping("/queryDataSourceConfig")
public Result queryDataSourceConfig() {
return Result.success(gfsDataConfig);
}
@RequestMapping("/updateDataSourceConfig")
public Result updateDataSourceConfig(@RequestParam String secret, @Validated @RequestBody GfsDataConfig updateConfig) {
validSecret(secret);
if (updateConfig == null) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空");
}
updateConfig.valid();
try {
gfsDataConfig.writeConfig(updateConfig);
} catch (IOException e) {
throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e);
}
return Result.success();
}
@RequestMapping("/download")
public Result download(@RequestParam String secret) {
validSecret(secret);
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.downloadAll(fileInfoList);
return Result.success(fileInfoList);
} catch (Exception e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
}
@RequestMapping("/import")
public Result importData(@RequestBody Map<String, String> params) {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"));
try {
gfsDataImport.importData(time);
return Result.success();
} catch (Exception e) {
e.printStackTrace();
return Result.error(e);
}
}
}

35
weather-service/src/main/java/com/htfp/weather/web/controller/ControllerExceptionAdvice.java
Unescape Escape View File

@ -0,0 +1,35 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.Result;
import org.springframework.validation.BindException;
import org.springframework.validation.ObjectError;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
/**
* @Author : shiyi
* @Date : 2024/3/14 16:28
* @Description : 校验异常捕获
*/
@RestControllerAdvice
public class ControllerExceptionAdvice {
@ExceptionHandler({BindException.class})
public Result methodArgumentNotValidExceptionHandler(BindException e) {
// 从异常对象中拿到ObjectError对象
ObjectError objectError = e.getBindingResult().getAllErrors().get(0);
return Result.error(ErrorCode.VALIDATE_ERROR, objectError.getDefaultMessage());
}
// @ExceptionHandler({Exception.class})
// public Result otherExceptionHandler(Exception e) {
// return Result.error(e);
// }
@ExceptionHandler({AppExcpetion.class})
public Result appExceptionHandler(AppExcpetion e) {
return Result.error(e.getErrorCode(), e.getMessage());
}
}

57
weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java
Unescape Escape View File

@ -0,0 +1,57 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.web.pojo.hefeng.HeFengWarningResponse;
import com.htfp.weather.web.pojo.request.Position2D;
import com.htfp.weather.web.pojo.response.NowWeatherStatus;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.service.surfaceapi.CaiYunServiceImpl;
import com.htfp.weather.web.service.surfaceapi.HeFengServiceImpl;
import com.htfp.weather.web.service.IDataService;
import com.htfp.weather.web.service.surfaceapi.ISurfaceDataService;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/24 16:07
* @Description : 地面天气状态
*/
@CrossOrigin
@RestController
@RequestMapping("/htfp/weather/surface/")
public class SurfaceWeatherController {
@Resource(name = "tablestore-gfs")
IDataService dataService;
@Resource(name = "caiyun")
ISurfaceDataService surfaceDataService;
@PostMapping("/now")
public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
NowWeatherStatus nowWeatherStatus = surfaceDataService.getNowSurfaceWeatherStatus(lat, lon);
return Result.success(nowWeatherStatus);
}
@PostMapping("/forecast")
public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
TimeSeriesDataset forecastSeries = surfaceDataService.getForecastSeries(lat, lon);
return Result.success(forecastSeries);
}
@PostMapping ("/warning")
public Result warning(@Validated @RequestBody Position2D position2D) throws Exception {
double lat = position2D.getLatitude();
double lon = position2D.getLongitude();
List<SurfaceWeatherWarning> warning = surfaceDataService.getSurfaceWarning(lat, lon);
return Result.success(warning);
}
}

30
weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java
Unescape Escape View File

@ -0,0 +1,30 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.griddata.common.TableConfigBean;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
/**
* @Author : shiyi
* @Date : 2024/4/20 17:30
* @Description :
*/
@Controller
public class ThymeleafTest {
@Resource
TableConfigBean tableConfigBean;
@GetMapping("index")// 页面的url地址
public String getindex(Model model, String secret) {
if (!"shiyi".equals(secret)){
return "wrong";
}
model.addAttribute("tableConfig", tableConfigBean);
return "index";// 与templates中index.html对应
}
}

65
weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java
Unescape Escape View File

@ -0,0 +1,65 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.pojo.request.PlaneRequest;
import com.htfp.weather.web.pojo.request.Position3D;
import com.htfp.weather.web.pojo.request.ProfileRequest;
import com.htfp.weather.web.pojo.response.PlaneResponse;
import com.htfp.weather.web.pojo.response.ProfileResponse;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.pojo.response.TimeSeriesDataset;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.time.OffsetDateTime;
/**
* @Author : shiyi
* @Date : 2024/5/8 15:15
* @Description :
*/
@CrossOrigin
@RestController
@RequestMapping("/htfp/weather/upper/")
public class UpperWeatherController {
@Resource(name = "tablestore-gfs")
GfsDataServiceImpl tablestoreService;
@RequestMapping("/profileByPressure")
public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) {
OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
String variableName = profileRequest.getVariableName();
double latitude = profileRequest.getLatitude();
double longitude = profileRequest.getLongitude();
ProfileResponse profileResponse = tablestoreService.getProfile(utcDateTime, variableName, latitude, longitude);
return Result.success(profileResponse);
}
@PostMapping("/forecast")
public Result queryTimeSeries(@Validated @RequestBody Position3D position3D) {
double latitude = position3D.getLatitude();
double longitude = position3D.getLongitude();
int level = position3D.getLevel();
TimeSeriesDataset forecastSeries = tablestoreService.getForecastSeries(latitude, longitude, level);
return Result.success(forecastSeries);
}
@PostMapping("/plane")
public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) {
planeRequest.valid();
String variableName = planeRequest.getVariableName();
OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime());
OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time);
int level = planeRequest.getLevel();
double minLat = planeRequest.getMinLat();
double maxLat = planeRequest.getMaxLat();
double minLon = planeRequest.getMinLon();
double maxLon = planeRequest.getMaxLon();
PlaneResponse forecastSeries = tablestoreService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon);
return Result.success(forecastSeries);
}
}

25
weather-service/src/main/java/com/htfp/weather/web/exception/AppExcpetion.java
Unescape Escape View File

@ -0,0 +1,25 @@
package com.htfp.weather.web.exception;
/**
* @Author : shiyi
* @Date : 2024/4/16 17:21
* @Description : 自定义异常
*/
public class AppExcpetion extends RuntimeException{
ErrorCode errorCode;
public ErrorCode getErrorCode() {
return errorCode;
}
public AppExcpetion(ErrorCode errorCode) {
this.errorCode = errorCode;
}
public AppExcpetion(ErrorCode errorCode, String message) {
super(message);
this.errorCode = errorCode;
}
public AppExcpetion(ErrorCode errorCode, Throwable e) {
super(e);
this.errorCode = errorCode;
}
}

45
weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
Unescape Escape View File

@ -0,0 +1,45 @@
package com.htfp.weather.web.exception;
/**
* @Author : shiyi
* @Date : 2024/3/14 16:30
* @Description : 错误码枚举
*/
public enum ErrorCode {
//
VALIDATE_ERROR(1001, "参数校验错误"),
CONFIG_ERROR(1002, "配置相关错误"),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),
CAI_YUN_DATA_PARSE_ERROR(4003, "数据解析错误"),
IMPORT_DATA_INITIAL_FAILED(5001, "导入数据初始化失败"),
NO_NC_OR_GRIB_FILES(5002, "文件夹不存在或中没有对应的气象数据文件"),
DATA_SET_EMPTY(6000, "TableStore数据库为空，无法查询"),
QUERY_TIME_ERROR(6001, "目标时间不在当前数据范围内"),
LONGITUDE_INDEX_ERROR(6002, "经度坐标索引"),
LATITUDE_INDEX_ERROR(6003, "纬度坐标索引"),
LEVEL_INDEX_ERROR(6004, "高度坐标索引"),
;
private final int code;
private final String msg;
ErrorCode(int code, String msg) {
this.code = code;
this.msg = msg;
}
public int getCode() {
return code;
}
public String getMsg() {
return msg;
}
}

44
weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunBaseResponse.java
Unescape Escape View File

@ -0,0 +1,44 @@
package com.htfp.weather.web.pojo.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/4/15 18:52
* @Description :
*/
@Data
public class CaiYunBaseResponse {
@JsonProperty("status")
private String status;
@JsonProperty("api_version")
private String apiVersion;
@JsonProperty("api_status")
private String apiStatus;
@JsonProperty("lang")
private String lang;
@JsonProperty("unit")
private String unit;
@JsonProperty("tzshift")
private int tzshift;
@JsonProperty("timezone")
private String timeZone;
@JsonProperty("server_time")
private long serverTime;
@JsonProperty("location")
private List<Double> location;
}

56
weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunForecastResponse.java
Unescape Escape View File

@ -0,0 +1,56 @@
package com.htfp.weather.web.pojo.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/4/15 18:52
* @Description :
*/
@Data
public class CaiYunForecastResponse extends CaiYunBaseResponse{
@JsonProperty("result")
private HourlyResult hourlyResult;
// Getters and Setters
@Data
public static class HourlyResult {
private Hourly hourly;
private int primary;
@JsonProperty("forecast_keypoint")
private String forecastKeypoint;
@Data @JsonIgnoreProperties(value = {"skycon", "apparent_temperature","dswrf", "air_quality"}, ignoreUnknown = true)
public static class Hourly {
private String status;
private String description;
private List<BaseData> precipitation;
private List<BaseData> temperature;
private List<Wind> wind;
private List<BaseData> humidity;
private List<BaseData> cloudrate;
private List<BaseData> pressure;
private List<BaseData> visibility;
}
@Data @JsonIgnoreProperties(value = {"probability"}, ignoreUnknown = true)
public static class BaseData {
private String datetime;
private Float value;
}
@Data
public static class Wind {
private String datetime;
private Float speed;
private Float direction;
}
}
}

63
weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunNowResponse.java
Unescape Escape View File

@ -0,0 +1,63 @@
package com.htfp.weather.web.pojo.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @Author : shiyi
* @Date : 2024/4/15 18:52
* @Description :
*/
@Data
public class CaiYunNowResponse extends CaiYunBaseResponse{
@JsonProperty("result")
private NowResult nowResult;
// Getters and Setters
@Data
public static class NowResult {
private RealTime realtime;
private int primary;
@Data @JsonIgnoreProperties(value = {"apparent_temperature","dswrf", "air_quality", "life_index"}, ignoreUnknown = true)
public static class RealTime {
private String status;
private String description;
private Precipitation precipitation;
private Float temperature;
private Wind wind;
private Float humidity;
private Float cloudrate;
private Float pressure;
private Float visibility;
private String skycon;
}
@Data
public static class Wind {
private Float speed;
private Float direction;
}
@Data
public static class Precipitation {
private Local local;
private Nearest nearest;
@Data
public static class Local {
private String status;
private String datasource;
private Float intensity;
}
@Data
public static class Nearest {
private String status;
private Float distance;
private Float intensity;
}
}
}
}

49
weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunWarningResponse.java
Unescape Escape View File

@ -0,0 +1,49 @@
package com.htfp.weather.web.pojo.caiyun;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/4/15 18:52
* @Description :
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class CaiYunWarningResponse extends CaiYunBaseResponse {
@JsonProperty("result")
private AlertResult result;
@Data @JsonIgnoreProperties(value = {"realtime"}, ignoreUnknown = true)
public static class AlertResult{
Alert alert;
@Data @JsonIgnoreProperties(value = {"abcodes"}, ignoreUnknown = true)
public static class Alert {
String status;
List<AlertContent> content;
@Data
public static class AlertContent {
private String province;
private String status;
private String code;
private String description;
private String regionId;
private String county;
private Long pubtimestamp;
private Float[] latlon;
private String city;
private String alertId;
private String title;
private String adcode;
private String source;
private String location;
@JsonProperty("request_status")
private String requestStatus;
}
}
}
}

24
weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/BaseHeFengData.java
Unescape Escape View File

@ -0,0 +1,24 @@
package com.htfp.weather.web.pojo.hefeng;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/1/24 15:23
* @Description :
*/
@Data
public class BaseHeFengData {
Float temp; // 温度; // 温度
String icon; // 天气状况和图标的代码，图标可通过"icon"字段来调用
String text; // 天气状况的文字描述，包括阴晴雨雪等天气状态的描述
Float wind360; // 风向360角度
String windDir; // 风向
String windScale; // 风力等级
Float windSpeed; // 风速，公里/小时
Float humidity; // 相对湿度
Float precip; // 当前小时累计降水量，默认单位：毫米
Float pressure; // 大气压强，默认单位：百帕
Float cloud; // 云量
Float dew; // 露点温度
}

14
weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengBaseResponse.java
Unescape Escape View File

@ -0,0 +1,14 @@
package com.htfp.weather.web.pojo.hefeng;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/4/17 15:04
* @Description :
*/
@Data
public class HeFengBaseResponse {
String code;
String updateTime;
}

26
weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengForecastResponse.java
Unescape Escape View File

@ -0,0 +1,26 @@
package com.htfp.weather.web.pojo.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.jetbrains.annotations.NotNull;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/23 18:01
* @Description : 和风天气预报响应
*/
@EqualsAndHashCode(callSuper = true)
@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true)
public class HeFengForecastResponse extends HeFengBaseResponse {
List<HeFengForecastHour> hourly;
@Data @EqualsAndHashCode(callSuper = true)
public static class HeFengForecastHour extends BaseHeFengData {
String fxTime; // 预报时间
}
}

21
weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengNowResponse.java
Unescape Escape View File

@ -0,0 +1,21 @@
package com.htfp.weather.web.pojo.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @Author : shiyi
* @Date : 2024/1/23 18:00
* @Description : 和风实时天气响应
*/
@EqualsAndHashCode(callSuper = true)
@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true)
public class HeFengNowResponse extends HeFengBaseResponse{
HeFengNow now;
@Data @EqualsAndHashCode(callSuper = true)
public static class HeFengNow extends BaseHeFengData {
String obsTime; // 观测时间
}
}

38
weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengWarningResponse.java
Unescape Escape View File

@ -0,0 +1,38 @@
package com.htfp.weather.web.pojo.hefeng;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/1/23 18:00
* @Description : 和风实时天气响应
*/
@EqualsAndHashCode(callSuper = true)
@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true)
public class HeFengWarningResponse extends HeFengBaseResponse{
List<HeFengWarning> warning;
@Data
public static class HeFengWarning{
private String id;
private String sender;
private String pubTime;
private String title;
private String startTime;
private String endTime;
private String status;
private String level;
private String severity;
private String severityColor;
private String type;
private String typeName;
private String urgency;
private String certainty;
private String text;
private String related;
}
}

14
weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java
Unescape Escape View File

@ -0,0 +1,14 @@
package com.htfp.weather.web.pojo.request;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/4/21 18:34
* @Description :
*/
@Data
public class ConfigUpdate {
String secret;
}

33
weather-service/src/main/java/com/htfp/weather/web/pojo/request/ForecastRequest.java
Unescape Escape View File

@ -0,0 +1,33 @@
package com.htfp.weather.web.pojo.request;
import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
/**
* @Author : shiyi
* @Date : 2024/3/14 18:01
* @Description : 预报请求参数
*/
@Data
public class ForecastRequest {
@NotNull(message = "变量名不能为空")
String variableName;
@NotNull(message = "高度层次不能为空")
Integer level;
@NotNull(message = "纬度不能为空")
@Min(value = -90, message = "纬度最小值为-90（南纬90°）")
@Max(value = 90, message = "纬度最大值为90（北纬90°）")
Double latitude;
@NotNull(message = "经度不能为空")
@Min(value = -180, message = "经度最小值为-180（西经180°）")
@Max(value = 180, message = "经度最大值为180（东经180°）")
Double longitude;
}

66
weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java
Unescape Escape View File

@ -0,0 +1,66 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.griddata.common.GfsVariableNameEnum;
import com.htfp.weather.info.Constant;
import com.htfp.weather.info.GfsDownloadVariableEnum;
import com.htfp.weather.info.GfsLevelsEnum;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.valid.DateTimeStr;
import com.htfp.weather.web.valid.EnumValid;
import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
/**
* @Author : shiyi
* @Date : 2024/5/15 16:11
* @Description : 平面数据请求体
*/
@Data
public class PlaneRequest {
@NotNull(message = "时间不能为空")
@DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx")
String time;
@NotNull(message = "变量名不能为空")
@EnumValid(enumClass = GfsVariableNameEnum.class, usedField = "getNameInApi", message = "变量不存在")
String variableName;
@NotNull(message = "高度层不能为空")
@EnumValid(enumClass = GfsLevelsEnum.class, usedField = "getCode", message = "高度层不存在")
Integer level;
@NotNull
@Min(value = 70, message = "经度最小值为70（东经180°）")
@Max(value = 140, message = "经度最大值为140（东经140°）")
private Double minLon;
@NotNull
@Min(value = 70, message = "经度最小值为70（东经180°）")
@Max(value = 140, message = "经度最大值为140（东经140°）")
private Double maxLon;
@NotNull
@Min(value = 0, message = "纬度最小值为0（0°）")
@Max(value = 55, message = "纬度最大值为55（北纬55°）")
private Double minLat;
@NotNull
@Min(value = 0, message = "纬度最小值为0（0°）")
@Max(value = 55, message = "纬度最大值为55（北纬55°）")
private Double maxLat;
public void valid() {
// TODO 2024/4/21: 校验
if (minLon >= maxLon) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度");
}
if (minLat >= maxLat) {
throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度");
}
}
}

26
weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position2D.java
Unescape Escape View File

@ -0,0 +1,26 @@
package com.htfp.weather.web.pojo.request;
import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
/**
* @Author : shiyi
* @Date : 2024/1/24 16:11
* @Description : 地面位置参数
*/
@Data
public class Position2D {
@NotNull(message = "纬度 (latitude) 不能为空")
@Min(value = -90, message = "纬度 (latitude) 最小值为-90（南纬90°）")
@Max(value = 90, message = "纬度 (latitude) 最大值为90（北纬90°）")
Double latitude;
@NotNull(message = "经度 (longitude) 不能为空")
@Min(value = -180, message = "经度 (longitude) 最小值为-180（西经180°）")
@Max(value = 180, message = "经度 (longitude) 最大值为180（东经180°）")
Double longitude;
}

20
weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position3D.java
Unescape Escape View File

@ -0,0 +1,20 @@
package com.htfp.weather.web.pojo.request;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @Author : shiyi
* @Date : 2024/1/24 16:11
* @Description : 地面位置参数
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class Position3D extends Position2D {
@NotNull(message = "高度层次 (level) 不能为空")
Integer level;
}

34
weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java
Unescape Escape View File

@ -0,0 +1,34 @@
package com.htfp.weather.web.pojo.request;
import com.htfp.weather.info.Constant;
import com.htfp.weather.web.valid.DateTimeStr;
import lombok.Data;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
/**
* @Author : shiyi
* @Date : 2024/5/10 17:27
* @Description : 廓线数据请求
*/
@Data
public class ProfileRequest {
@NotNull(message = "时间不能为空")
@DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx")
String time;
@NotNull(message = "变量名不能为空")
String variableName;
@NotNull(message = "纬度 (latitude) 不能为空")
@Min(value = -90, message = "纬度 (latitude) 最小值为-90（南纬90°）")
@Max(value = 90, message = "纬度 (latitude) 最大值为90（北纬90°）")
double latitude;
@NotNull(message = "经度 (longitude) 不能为空")
@Min(value = -180, message = "经度 (longitude) 最小值为-180（西经180°）")
@Max(value = 180, message = "经度 (longitude) 最大值为180（东经180°）")
double longitude;
}

23
weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java
Unescape Escape View File

@ -0,0 +1,23 @@
package com.htfp.weather.web.pojo.response;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/1/24 11:28
* @Description : 当前天气状况
*/
@Data
public class NowWeatherStatus {
String time; // 数据时间，北京时
String weatherText; // 天气状况
float temp; // 温度; // 温度
float windSpeed; // 风速，m/s
float wind360; // 风向360角度
// String windDir; // 风向, 文字描述，如“西北风”
// String windScale; // 风力等级
float humidity; // 相对湿度 %
// float pressure; // 大气压强，默认单位：百帕
float cloud; // 云量
float precip; //降水量
}

Some files were not shown because too many files have changed in this diff Show More

Write Preview
Loading…
Cancel
Save