From bc43f9f1cade51d1c2da51af2c41f4bf3bfafe88 Mon Sep 17 00:00:00 2001 From: shiyi Date: Mon, 20 May 2024 11:15:15 +0800 Subject: [PATCH] first commit --- .gitignore | 23 ++ tablestore-grid-master/pom.xml | 122 ++++++++ .../example/grid/CreateStoreExample.java | 54 ++++ .../example/grid/DataFetchExample.java | 82 ++++++ .../example/grid/DataImportExample.java | 110 +++++++ .../example/grid/MetaDeleteExample.java | 59 ++++ .../example/grid/MetaQueryExample.java | 116 ++++++++ .../example/grid/common/ExampleConfig.java | 24 ++ .../example/grid/common/TableStoreConf.java | 62 ++++ .../grid/common/TableStoreGridExample.java | 33 +++ .../tablestore/grid/GridDataDeleter.java | 10 + .../tablestore/grid/GridDataFetcher.java | 47 +++ .../tablestore/grid/GridDataWriter.java | 15 + .../com/aliyun/tablestore/grid/GridStore.java | 79 +++++ .../tablestore/grid/TableStoreGrid.java | 191 ++++++++++++ .../tablestore/grid/TableStoreGridConfig.java | 70 +++++ .../grid/consts/AttributionEnum.java | 29 ++ .../tablestore/grid/consts/Constants.java | 26 ++ .../tablestore/grid/core/QueryBuilder.java | 134 +++++++++ .../tablestore/grid/core/RequestBuilder.java | 149 ++++++++++ .../tablestore/grid/core/RowParser.java | 75 +++++ .../grid/core/TableStoreDataDeleter.java | 68 +++++ .../grid/core/TableStoreDataFetcher.java | 224 ++++++++++++++ .../grid/core/TableStoreDataWriter.java | 98 +++++++ .../grid/model/DeleteDataParam.java | 20 ++ .../tablestore/grid/model/GetDataParam.java | 70 +++++ .../tablestore/grid/model/GridDataSet.java | 42 +++ .../grid/model/GridDataSetMeta.java | 124 ++++++++ .../grid/model/QueryGridDataSetResult.java | 43 +++ .../tablestore/grid/model/QueryParams.java | 82 ++++++ .../tablestore/grid/model/StoreOptions.java | 41 +++ .../tablestore/grid/model/grid/Grid.java | 58 ++++ .../tablestore/grid/model/grid/Grid2D.java | 19 ++ .../tablestore/grid/model/grid/Grid3D.java | 30 ++ .../tablestore/grid/model/grid/Grid4D.java | 30 ++ .../tablestore/grid/model/grid/Plane.java | 55 ++++ .../tablestore/grid/model/grid/Point.java | 39 +++ .../tablestore/grid/model/grid/Range.java | 53 ++++ .../tablestore/grid/utils/BlockUtil.java | 90 ++++++ .../tablestore/grid/utils/ValueUtil.java | 47 +++ .../src/main/resources/META-INF/MANIFEST.MF | 13 + .../src/main/resources/logback.xml | 21 ++ .../src/test/java/TestReadGrib2ByNC.java | 60 ++++ weather-service/pom.xml | 159 ++++++++++ .../weather/WeatherServiceApplication.java | 19 ++ .../java/com/htfp/weather/config/Config.java | 20 ++ .../weather/download/BaseDataDownloader.java | 51 ++++ .../com/htfp/weather/download/FileInfo.java | 31 ++ .../htfp/weather/download/GfsDataConfig.java | 154 ++++++++++ .../htfp/weather/download/GfsDownloader.java | 265 +++++++++++++++++ .../griddata/common/GfsVariableNameEnum.java | 54 ++++ .../weather/griddata/common/TableConfig.java | 108 +++++++ .../griddata/common/TableConfigBean.java | 144 +++++++++ .../griddata/common/TableStoreConf.java | 15 + .../operation/BaseTableOperation.java | 44 +++ .../griddata/operation/CreateTable.java | 53 ++++ .../griddata/operation/DataDeleter.java | 130 +++++++++ .../griddata/operation/GfsDataFetcher.java | 137 +++++++++ .../griddata/operation/GfsDataImport.java | 264 +++++++++++++++++ .../griddata/operation/IDataFetch.java | 9 + .../weather/griddata/operation/QueryMeta.java | 51 ++++ .../griddata/operation/UpdateTable.java | 29 ++ .../griddata/utils/CoordinateUtils.java | 45 +++ .../htfp/weather/griddata/utils/GfsUtils.java | 39 +++ .../com/htfp/weather/info/CaiYunInfo.java | 81 ++++++ .../java/com/htfp/weather/info/Constant.java | 14 + .../weather/info/GfsDownloadVariableEnum.java | 42 +++ .../com/htfp/weather/info/GfsLevelsEnum.java | 54 ++++ .../weather/schedule/GridDataProcesser.java | 76 +++++ .../com/htfp/weather/utils/DateTimeUtils.java | 135 +++++++++ .../com/htfp/weather/utils/DownloadUtils.java | 9 + .../htfp/weather/utils/HttpClientUtils.java | 86 ++++++ .../com/htfp/weather/utils/JSONUtils.java | 92 ++++++ .../com/htfp/weather/utils/MeteoUtils.java | 144 +++++++++ .../com/htfp/weather/utils/NdArrayUtils.java | 42 +++ .../com/htfp/weather/utils/SpringUtil.java | 46 +++ .../htfp/weather/web/config/CORSFilter.java | 23 ++ .../web/controller/ConfigController.java | 113 ++++++++ .../controller/ControllerExceptionAdvice.java | 35 +++ .../controller/SurfaceWeatherController.java | 57 ++++ .../weather/web/controller/ThymeleafTest.java | 30 ++ .../controller/UpperWeatherController.java | 65 +++++ .../weather/web/exception/AppExcpetion.java | 25 ++ .../htfp/weather/web/exception/ErrorCode.java | 45 +++ .../web/pojo/caiyun/CaiYunBaseResponse.java | 44 +++ .../pojo/caiyun/CaiYunForecastResponse.java | 56 ++++ .../web/pojo/caiyun/CaiYunNowResponse.java | 63 ++++ .../pojo/caiyun/CaiYunWarningResponse.java | 49 ++++ .../web/pojo/hefeng/BaseHeFengData.java | 24 ++ .../web/pojo/hefeng/HeFengBaseResponse.java | 14 + .../pojo/hefeng/HeFengForecastResponse.java | 26 ++ .../web/pojo/hefeng/HeFengNowResponse.java | 21 ++ .../pojo/hefeng/HeFengWarningResponse.java | 38 +++ .../web/pojo/request/ConfigUpdate.java | 14 + .../web/pojo/request/ForecastRequest.java | 33 +++ .../web/pojo/request/PlaneRequest.java | 66 +++++ .../weather/web/pojo/request/Position2D.java | 26 ++ .../weather/web/pojo/request/Position3D.java | 20 ++ .../web/pojo/request/ProfileRequest.java | 34 +++ .../web/pojo/response/NowWeatherStatus.java | 23 ++ .../web/pojo/response/PlaneResponse.java | 15 + .../web/pojo/response/ProfileResponse.java | 21 ++ .../weather/web/pojo/response/Result.java | 46 +++ .../pojo/response/SurfaceWeatherWarning.java | 19 ++ .../weather/web/pojo/response/TimeSeries.java | 17 ++ .../web/pojo/response/TimeSeriesDataset.java | 37 +++ .../web/service/GfsDataServiceImpl.java | 249 ++++++++++++++++ .../weather/web/service/IDataService.java | 9 + .../service/surfaceapi/CaiYunServiceImpl.java | 172 +++++++++++ .../service/surfaceapi/HeFengServiceImpl.java | 273 ++++++++++++++++++ .../surfaceapi/ISurfaceDataService.java | 20 ++ .../htfp/weather/web/valid/DateTimeStr.java | 28 ++ .../weather/web/valid/DateTimeValidator.java | 45 +++ .../com/htfp/weather/web/valid/EnumValid.java | 27 ++ .../htfp/weather/web/valid/EnumValidator.java | 63 ++++ .../src/main/resources/application-dev.yml | 6 + .../main/resources/application-weather.yml | 12 + .../src/main/resources/application.yml | 12 + .../main/resources/config/gfsDataConfig.json | 11 + .../src/main/resources/config/tableConf.json | 12 + .../src/main/resources/logback.xml | 45 +++ .../src/main/resources/static/index.html | 6 + .../src/main/resources/templates/index.html | 60 ++++ .../src/main/resources/templates/wrong.html | 10 + .../WeatherServiceApplicationTests.java | 20 ++ .../weather/download/GfsDataConfigTest.java | 24 ++ .../weather/download/GfsDownloaderTest.java | 54 ++++ .../operation/GfsDataFetcherTest.java | 42 +++ .../griddata/operation/GfsDataImportTest.java | 52 ++++ .../griddata/operation/QueryMetaTest.java | 30 ++ .../weather/griddata/utils/GfsUtilsTest.java | 27 ++ .../schedule/GridDataProcesserTest.java | 23 ++ .../htfp/weather/utils/DateTimeUtilsTest.java | 49 ++++ .../weather/utils/HttpClientUtilsTest.java | 93 ++++++ .../htfp/weather/utils/MeteoUtilsTest.java | 57 ++++ .../htfp/weather/utils/NdArrayUtilsTest.java | 82 ++++++ .../UpperWeatherControllerTest.java | 17 ++ .../surfaceapi/CaiYunServiceImplTest.java | 37 +++ .../surfaceapi/HeFengServiceImplTest.java | 50 ++++ .../src/test/resources/application-dev.yml | 6 + .../test/resources/application-weather.yml | 12 + .../src/test/resources/application.yml | 12 + .../test/resources/config/gfsDataConfig.json | 11 + .../src/test/resources/config/tableConf.json | 12 + .../src/test/resources/logback-test.xml | 47 +++ .../src/test/resources/static/index.html | 6 + 146 files changed, 8401 insertions(+) create mode 100644 .gitignore create mode 100644 tablestore-grid-master/pom.xml create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/CreateStoreExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataFetchExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataImportExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaDeleteExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaQueryExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/ExampleConfig.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreConf.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreGridExample.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataDeleter.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataFetcher.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataWriter.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridStore.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGrid.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGridConfig.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/AttributionEnum.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/Constants.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/QueryBuilder.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RequestBuilder.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RowParser.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataDeleter.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataFetcher.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataWriter.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/DeleteDataParam.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GetDataParam.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSet.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSetMeta.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryGridDataSetResult.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryParams.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/StoreOptions.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid2D.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid3D.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid4D.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Plane.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Point.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Range.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/BlockUtil.java create mode 100644 tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/ValueUtil.java create mode 100644 tablestore-grid-master/src/main/resources/META-INF/MANIFEST.MF create mode 100644 tablestore-grid-master/src/main/resources/logback.xml create mode 100644 tablestore-grid-master/src/test/java/TestReadGrib2ByNC.java create mode 100644 weather-service/pom.xml create mode 100644 weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java create mode 100644 weather-service/src/main/java/com/htfp/weather/config/Config.java create mode 100644 weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java create mode 100644 weather-service/src/main/java/com/htfp/weather/download/FileInfo.java create mode 100644 weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java create mode 100644 weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/common/TableStoreConf.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/utils/CoordinateUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/griddata/utils/GfsUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/info/CaiYunInfo.java create mode 100644 weather-service/src/main/java/com/htfp/weather/info/Constant.java create mode 100644 weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java create mode 100644 weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java create mode 100644 weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/DownloadUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/JSONUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/NdArrayUtils.java create mode 100644 weather-service/src/main/java/com/htfp/weather/utils/SpringUtil.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/config/CORSFilter.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/controller/ControllerExceptionAdvice.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/exception/AppExcpetion.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunBaseResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunForecastResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunNowResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunWarningResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/BaseHeFengData.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengBaseResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengForecastResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengNowResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengWarningResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/ForecastRequest.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position2D.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position3D.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/PlaneResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileResponse.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/SurfaceWeatherWarning.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeries.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/IDataService.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/ISurfaceDataService.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeStr.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeValidator.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/valid/EnumValid.java create mode 100644 weather-service/src/main/java/com/htfp/weather/web/valid/EnumValidator.java create mode 100644 weather-service/src/main/resources/application-dev.yml create mode 100644 weather-service/src/main/resources/application-weather.yml create mode 100644 weather-service/src/main/resources/application.yml create mode 100644 weather-service/src/main/resources/config/gfsDataConfig.json create mode 100644 weather-service/src/main/resources/config/tableConf.json create mode 100644 weather-service/src/main/resources/logback.xml create mode 100644 weather-service/src/main/resources/static/index.html create mode 100644 weather-service/src/main/resources/templates/index.html create mode 100644 weather-service/src/main/resources/templates/wrong.html create mode 100644 weather-service/src/test/java/com/htfp/weather/WeatherServiceApplicationTests.java create mode 100644 weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataImportTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/operation/QueryMetaTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/utils/DateTimeUtilsTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/utils/HttpClientUtilsTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/utils/MeteoUtilsTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/utils/NdArrayUtilsTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/web/controller/UpperWeatherControllerTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImplTest.java create mode 100644 weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImplTest.java create mode 100644 weather-service/src/test/resources/application-dev.yml create mode 100644 weather-service/src/test/resources/application-weather.yml create mode 100644 weather-service/src/test/resources/application.yml create mode 100644 weather-service/src/test/resources/config/gfsDataConfig.json create mode 100644 weather-service/src/test/resources/config/tableConf.json create mode 100644 weather-service/src/test/resources/logback-test.xml create mode 100644 weather-service/src/test/resources/static/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ffd1490 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +/GFSData/ +/tablestoreConf.json +/log/ +/*/target/ +/tablestore-grid-master/tableConf.json +/tablestore-grid-master/tablestoreConf.json +/tablestore-grid-master/out/ + + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr diff --git a/tablestore-grid-master/pom.xml b/tablestore-grid-master/pom.xml new file mode 100644 index 0000000..fe13af0 --- /dev/null +++ b/tablestore-grid-master/pom.xml @@ -0,0 +1,122 @@ + + + 4.0.0 + + com.aliyun.tablestore + tablestore-grid + 1.0-SNAPSHOT + + + + unidata-all + Unidata All + https://artifacts.unidata.ucar.edu/repository/unidata-all/ + + + + + + edu.ucar + cdm-core + 5.5.3 + + + edu.ucar + grib + 5.5.3 + + + com.aliyun.openservices + tablestore + 5.16.0 + jar-with-dependencies + + + org.apache.httpcomponents + httpasyncclient + + + + + commons-io + commons-io + 2.11.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + + ch.qos.logback + logback-classic + 1.4.12 + + + org.projectlombok + lombok + 1.18.30 + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + \ No newline at end of file diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/CreateStoreExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/CreateStoreExample.java new file mode 100644 index 0000000..7adcb37 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/CreateStoreExample.java @@ -0,0 +1,54 @@ +package com.aliyun.tablestore.example.grid; + +import com.alicloud.openservices.tablestore.model.search.FieldSchema; +import com.alicloud.openservices.tablestore.model.search.FieldType; +import com.alicloud.openservices.tablestore.model.search.IndexSchema; +import com.aliyun.tablestore.example.grid.common.ExampleConfig; +import com.aliyun.tablestore.example.grid.common.TableStoreGridExample; + +import java.util.Arrays; + +/** + * this example will create table and index. + */ +public class CreateStoreExample extends TableStoreGridExample { + + public CreateStoreExample() { + super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME); + } + + /** + * we must create store before we can use it. + * @throws Exception + */ + private void createStore() throws Exception { + this.tableStoreGrid.createStore(); + } + + /** + * this example create an index which contains these columns: status, tag1, tag2, create_time. + * you can create an index which contains any other columns. + * + * @throws Exception + */ + private void createIndex() throws Exception { + IndexSchema indexSchema = new IndexSchema(); + indexSchema.setFieldSchemas(Arrays.asList( + new FieldSchema("status", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true), + new FieldSchema("tag1", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true), + new FieldSchema("tag2", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true), + new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true) + )); + this.tableStoreGrid.createMetaIndex(ExampleConfig.GRID_META_INDEX_NAME, indexSchema); + } + + public static void main(String[] args) throws Exception { + CreateStoreExample example = new CreateStoreExample(); + try { + example.createStore(); + example.createIndex(); + } finally { + example.close(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataFetchExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataFetchExample.java new file mode 100644 index 0000000..d897470 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataFetchExample.java @@ -0,0 +1,82 @@ +package com.aliyun.tablestore.example.grid; + +import com.aliyun.tablestore.example.grid.common.ExampleConfig; +import com.aliyun.tablestore.example.grid.common.TableStoreGridExample; +import com.aliyun.tablestore.grid.GridDataFetcher; +import com.aliyun.tablestore.grid.model.grid.Grid4D; +import ucar.ma2.Array; + +import java.util.Arrays; +import java.util.Collections; + +public class DataFetchExample extends TableStoreGridExample { + + public DataFetchExample() { + super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME); + } + + public Array queryByTableStore(String dataSetId, String variable, int[] origin, int[] shape) throws Exception { + GridDataFetcher fetcher = this.tableStoreGrid.getDataFetcher(this.tableStoreGrid.getDataSetMeta(dataSetId)); + fetcher.setVariablesToGet(Collections.singletonList(variable)); + fetcher.setOriginShape(origin, shape); + Grid4D grid4D = fetcher.fetch().getVariable(variable); + return grid4D.toArray(); + } + + /** + * get a plane. + * @throws Exception + */ + public void fetch1() throws Exception { + int[] origin = new int[] {0, 0, 0, 0}; + int[] shape = new int[] {1, 1, ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[2], ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[3]}; + Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID, + ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE, + origin, shape); + System.out.println("Shape: " + array.shapeToString()); + System.out.println("Data: " + array); + } + + /** + * get an sequence of point with different levels. + * @throws Exception + */ + public void fetch2() throws Exception { + int[] origin = new int[] {0, 0, 0, 0}; + int[] shape = new int[] {1, ExampleConfig.EXAMPLE_GRID_DATA_SHAPE[1], 1, 1}; + Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID, + ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE, + origin, shape); + System.out.println("Shape:" + array.shapeToString()); + System.out.println("Data:" + array); + } + + /** + * get arbitrary 4-dimension data. + * @throws Exception + */ + public void fetch3() throws Exception { + int[] origin = new int[] {2, 5, 10, 10}; + int[] shape = new int[] {3, 10, 30, 30}; + Array array = queryByTableStore(ExampleConfig.EXAMPLE_GRID_DATA_SET_ID, + ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE, + origin, shape); + System.out.println("Shape:" + array.shapeToString()); + System.out.println("Data:" + array); + } + + public void run() throws Exception { + fetch1(); + fetch2(); + fetch3(); + } + + public static void main(String[] args) throws Exception { + DataFetchExample example = new DataFetchExample(); + try { + example.run(); + } finally { + example.close(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataImportExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataImportExample.java new file mode 100644 index 0000000..ab78057 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/DataImportExample.java @@ -0,0 +1,110 @@ +package com.aliyun.tablestore.example.grid; + +import com.aliyun.tablestore.example.grid.common.ExampleConfig; +import com.aliyun.tablestore.example.grid.common.TableStoreGridExample; +import com.aliyun.tablestore.grid.GridDataWriter; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.StoreOptions; +import com.aliyun.tablestore.grid.model.grid.Grid2D; +import lombok.extern.slf4j.Slf4j; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@Slf4j +public class DataImportExample extends TableStoreGridExample { + + public DataImportExample() { + super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME); + } + + /** + * init meta data to table store. + * @param dataSetID + * @param dataType + * @param variables + * @param shape + * @return + * @throws Exception + */ + public GridDataSetMeta initMeta(String dataSetID, DataType dataType, List variables, int[] shape) throws Exception { + GridDataSetMeta meta = new GridDataSetMeta( + dataSetID, + dataType, + variables, + shape[0], + shape[1], + shape[2], + shape[3], + new StoreOptions(StoreOptions.StoreType.SLICE)); + meta.addAttribute("status", "INIT"); + meta.addAttribute("create_time", System.currentTimeMillis()); + tableStoreGrid.putDataSetMeta(meta); + return meta; + } + + /** + * update meta and set status to DONE when data import finished. + * @param meta + * @return + * @throws Exception + */ + public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception { + meta.addAttribute("status", "DONE"); + tableStoreGrid.updateDataSetMeta(meta); + return meta; + } + + /** + * read data from netcdf file and write data to table store. + * @param meta + * @param ncFileName + * @throws Exception + */ + public void importFromNcFile(GridDataSetMeta meta, String ncFileName) throws Exception { + GridDataWriter writer = tableStoreGrid.getDataWriter(meta); + NetcdfFile ncFile = NetcdfFile.open(ncFileName); + List variables = ncFile.getVariables(); + + for (Variable variable : variables) { + if (meta.getVariables().contains(variable.getShortName())) { + for (int t = 0; t < meta.gettSize(); t++) { + for (int z = 0; z < meta.getzSize(); z++) { + Array array = variable.read(new int[]{t, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()}); + Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(), + new int[] {0, 0}, new int[] {meta.getxSize(), meta.getySize()}); + writer.writeGrid2D(variable.getShortName(), t, z, grid2D); + } + } + } + } + } + + public void run() throws Exception { + String dataSetId = ExampleConfig.EXAMPLE_GRID_DATA_SET_ID; + String filePath = ExampleConfig.EXAMPLE_GRID_DATA_SET_NC_FILE_PATH; + String variable = ExampleConfig.EXAMPLE_GRID_DATA_VARIABLE; + int[] shape = ExampleConfig.EXAMPLE_GRID_DATA_SHAPE; + DataType dataType = ExampleConfig.EXAMPLE_GRID_DATA_TYPE; + + GridDataSetMeta meta = initMeta(dataSetId, dataType, Collections.singletonList(variable), shape); + importFromNcFile(meta, filePath); + updateMeta(meta); + } + + public static void main(String[] args) throws Exception { + DataImportExample example = new DataImportExample(); + try { + log.info("导入数据"); + example.run(); + } finally { + example.close(); + } + } + +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaDeleteExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaDeleteExample.java new file mode 100644 index 0000000..2bd5540 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaDeleteExample.java @@ -0,0 +1,59 @@ +package com.aliyun.tablestore.example.grid; + +import com.alicloud.openservices.tablestore.model.search.sort.FieldSort; +import com.alicloud.openservices.tablestore.model.search.sort.Sort; +import com.alicloud.openservices.tablestore.model.search.sort.SortOrder; +import com.aliyun.tablestore.example.grid.common.ExampleConfig; +import com.aliyun.tablestore.example.grid.common.TableStoreGridExample; +import com.aliyun.tablestore.grid.core.QueryBuilder; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; +import com.aliyun.tablestore.grid.model.QueryParams; +import lombok.extern.slf4j.Slf4j; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@Slf4j +public class MetaDeleteExample extends TableStoreGridExample { + public MetaDeleteExample() { + super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME); + } + + private List query() throws Exception { + QueryGridDataSetResult result = tableStoreGrid.queryDataSets( + ExampleConfig.GRID_META_INDEX_NAME, + QueryBuilder.or() + .equal("status", "DONE") + .equal("status", "INIT") + .build(), + new QueryParams(0, 1000, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC))))); + + return result.getGridDataSetMetas(); + } + + private void batchDeleteRow() throws Exception { + List metaList = query(); + while (!metaList.isEmpty()) { + for (GridDataSetMeta meta : metaList) { + // 获取行的主键 + String gridDataSetId = meta.getGridDataSetId(); + if (!"test_echam_spectral_example".equals(gridDataSetId)) { + log.info("删除行：{}", gridDataSetId); + tableStoreGrid.deleteDataSetMeta(gridDataSetId); + } + } + metaList = query(); + } + } + + public static void main(String[] args) throws Exception { + MetaDeleteExample example = new MetaDeleteExample(); + try { + example.batchDeleteRow(); + } finally { + example.close(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaQueryExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaQueryExample.java new file mode 100644 index 0000000..1e029b4 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/MetaQueryExample.java @@ -0,0 +1,116 @@ +package com.aliyun.tablestore.example.grid; + +import com.alicloud.openservices.tablestore.model.search.sort.FieldSort; +import com.alicloud.openservices.tablestore.model.search.sort.Sort; +import com.alicloud.openservices.tablestore.model.search.sort.SortOrder; +import com.aliyun.tablestore.example.grid.common.ExampleConfig; +import com.aliyun.tablestore.example.grid.common.TableStoreGridExample; +import com.aliyun.tablestore.grid.core.QueryBuilder; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; +import com.aliyun.tablestore.grid.model.QueryParams; +import com.aliyun.tablestore.grid.model.StoreOptions; +import ucar.ma2.DataType; + +import java.util.*; + +public class MetaQueryExample extends TableStoreGridExample { + + public MetaQueryExample() { + super(ExampleConfig.GRID_DATA_TABLE_NAME, ExampleConfig.GRID_META_TABLE_NAME); + } + + private GridDataSetMeta initMeta(String uniqueKey, DataType dataType, List variables, int[] shape, Map attributes) throws Exception { + GridDataSetMeta meta = new GridDataSetMeta( + uniqueKey, + dataType, + variables, + shape[0], + shape[1], + shape[2], + shape[3], + new StoreOptions(StoreOptions.StoreType.SLICE)); + meta.setAttributes(attributes); + tableStoreGrid.putDataSetMeta(meta); + return meta; + } + + /** + * write some metas to table store for test. + * @throws Exception + */ + private void prepareMetas() throws Exception { + for (String tagA : Arrays.asList("A", "B", "C")) { + for (String tagB : Arrays.asList("X", "Y", "Z")) { + for (int i = 0; i < 10; i++) { + String dataSetId = UUID.randomUUID().toString(); + Map objectMap = new HashMap(); + objectMap.put("create_time", System.currentTimeMillis()); + objectMap.put("tag1", tagA); + objectMap.put("tag2", tagB); + objectMap.put("status", ((i % 2) == 0)?"INIT":"DONE"); + initMeta(dataSetId, DataType.FLOAT, Collections.singletonList("var"), new int[]{1, 1, 1, 1}, objectMap); + } + } + } + } + + /** + * query by arbitrary combination conditions. + * @throws Exception + */ + private void query() throws Exception { + /** + * query condition : (status == DONE) and (create_time > System.currentTimeMillis - 86400000) and (tag1 == A or tag2 == X) + * sort by create_time, desc. + */ + QueryGridDataSetResult result = tableStoreGrid.queryDataSets( + ExampleConfig.GRID_META_INDEX_NAME, + QueryBuilder.and() + .equal("status", "DONE") + .greaterThan("create_time", System.currentTimeMillis() - 86400000) + // .equal("accuracy", "1km") + .query(QueryBuilder.or() + .equal("tag1", "A") + .equal("tag2", "X") + // .equal("source", "ECMWF") + // .equal("source", "NMC") + .build()) + .build(), + new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC))))); + + System.out.println("GetMetaCount: " + result.getGridDataSetMetas().size()); + for (GridDataSetMeta meta : result.getGridDataSetMetas()) { + System.out.println("Meta: " + meta.getGridDataSetId()); + System.out.println(meta.getAttributes()); + } + } + private void query1() throws Exception { + /** + * query condition : (status == DONE) and (create_time > System.currentTimeMillis - 86400000) and (tag1 == A or tag2 == X) + * sort by create_time, desc. + */ + QueryGridDataSetResult result = tableStoreGrid.queryDataSets( + "gfs_meta_table_index", + QueryBuilder.and() + .equal("status", "DONE") + .greaterThanEqual("reference_time", 1684368800000L) + .build(), + new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("create_time", SortOrder.DESC))))); + + System.out.println("GetMetaCount: " + result.getGridDataSetMetas().size()); + for (GridDataSetMeta meta : result.getGridDataSetMetas()) { + System.out.println("Meta: " + meta.getGridDataSetId()); + System.out.println(meta.getAttributes()); + } + } + public static void main(String[] args) throws Exception { + MetaQueryExample example = new MetaQueryExample(); + try { + System.out.println("Query..."); + example.query1(); + } finally { + example.close(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/ExampleConfig.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/ExampleConfig.java new file mode 100644 index 0000000..6b5eb52 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/ExampleConfig.java @@ -0,0 +1,24 @@ +package com.aliyun.tablestore.example.grid.common; + +import ucar.ma2.DataType; + + +public class ExampleConfig { + + /** + * 表名和index名 + */ + public static String GRID_DATA_TABLE_NAME = "GRID_DATA_TABLE_EXAMPLE1"; + public static String GRID_META_TABLE_NAME = "GRID_META_TABLE_EXAMPLE1"; + public static String GRID_META_INDEX_NAME = "GRID_META_INDEX"; + + + /** + * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 + */ + public static String EXAMPLE_GRID_DATA_SET_ID = "test_echam_spectral_example"; + public static String EXAMPLE_GRID_DATA_SET_NC_FILE_PATH = "test_echam_spectral.nc"; + public static String EXAMPLE_GRID_DATA_VARIABLE = "tpot"; + public static int[] EXAMPLE_GRID_DATA_SHAPE = new int[]{8, 47, 96, 192}; + public static DataType EXAMPLE_GRID_DATA_TYPE = DataType.FLOAT; +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreConf.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreConf.java new file mode 100644 index 0000000..b98964c --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreConf.java @@ -0,0 +1,62 @@ +package com.aliyun.tablestore.example.grid.common; + +import com.google.gson.Gson; +import org.apache.commons.io.IOUtils; + +import java.io.FileInputStream; +import java.io.InputStream; + + + +public class TableStoreConf { + + private String endpoint; + private String accessId; + private String accessKey; + private String instanceName; + + /** + * 从目标配置文件中读取 数据库instance相关信息 + */ + public static TableStoreConf newInstance(String path) { + try { + InputStream f = new FileInputStream(path); + Gson gson = new Gson(); + return gson.fromJson(IOUtils.toString(f), TableStoreConf.class); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getAccessId() { + return accessId; + } + + public void setAccessId(String accessId) { + this.accessId = accessId; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getInstanceName() { + return instanceName; + } + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreGridExample.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreGridExample.java new file mode 100644 index 0000000..cc03f78 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/example/grid/common/TableStoreGridExample.java @@ -0,0 +1,33 @@ +package com.aliyun.tablestore.example.grid.common; + +import com.aliyun.tablestore.grid.TableStoreGrid; +import com.aliyun.tablestore.grid.TableStoreGridConfig; + +public abstract class TableStoreGridExample { + + protected TableStoreGrid tableStoreGrid; + // private String pathSeperator = "/"; + + public TableStoreGridExample(String dataTableName, String metaTableName) { + // String os = System.getProperty("os.name"); + // if (os.toLowerCase().startsWith("win")) { + // pathSeparator = "\\"; + // } + String pathSeparator = System.getProperty("file.separator"); + TableStoreConf conf = TableStoreConf.newInstance(System.getProperty("user.dir") + pathSeparator + "tablestoreConf.json"); + TableStoreGridConfig config = new TableStoreGridConfig(); + config.setTableStoreEndpoint(conf.getEndpoint()); + config.setAccessId(conf.getAccessId()); + config.setAccessKey(conf.getAccessKey()); + config.setTableStoreInstance(conf.getInstanceName()); + config.setDataTableName(dataTableName); + config.setMetaTableName(metaTableName); + tableStoreGrid = new TableStoreGrid(config); + } + + public void close() { + if (tableStoreGrid != null) { + tableStoreGrid.close(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataDeleter.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataDeleter.java new file mode 100644 index 0000000..dcdeac4 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataDeleter.java @@ -0,0 +1,10 @@ +package com.aliyun.tablestore.grid; + +import com.aliyun.tablestore.grid.model.grid.Grid2D; + +import java.util.concurrent.ExecutionException; + +public interface GridDataDeleter { + + void delete(String dataSetId) throws ExecutionException, InterruptedException; +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataFetcher.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataFetcher.java new file mode 100644 index 0000000..2b69f70 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataFetcher.java @@ -0,0 +1,47 @@ +package com.aliyun.tablestore.grid; + +import com.aliyun.tablestore.grid.model.GridDataSet; +import com.aliyun.tablestore.grid.model.grid.Range; + +import java.util.Collection; + +public interface GridDataFetcher { + + /** + * 设置要查询的变量。 + * @param variables + * @return + */ + GridDataFetcher setVariablesToGet(Collection variables); + + /** + * 设置要读取的各维度起始点和大小。 + * @param origin 各维度起始点。 + * @param shape 各维度大小。 + * @return + */ + GridDataFetcher setOriginShape(int[] origin, int[] shape); + + /** + * 获取数据。 + * @return + * @throws Exception + */ + GridDataSet fetch() throws Exception; + + GridDataFetcher setT(int t); + + GridDataFetcher setTRange(Range tRange); + + GridDataFetcher setZ(int z); + + GridDataFetcher setZRange(Range zRange); + + GridDataFetcher setX(int x); + + GridDataFetcher setXRange(Range xRange); + + GridDataFetcher setY(int y); + + GridDataFetcher setYRange(Range yRange); +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataWriter.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataWriter.java new file mode 100644 index 0000000..6a3f1c0 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridDataWriter.java @@ -0,0 +1,15 @@ +package com.aliyun.tablestore.grid; + +import com.aliyun.tablestore.grid.model.grid.Grid2D; + +public interface GridDataWriter { + /** + * 写入一个二维平面。 + * @param variable 变量名。 + * @param t 时间维的值。 + * @param z 高度维的值。 + * @param grid2D 平面数据。 + * @throws Exception + */ + void writeGrid2D(String variable, int t, int z, Grid2D grid2D) throws Exception; +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridStore.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridStore.java new file mode 100644 index 0000000..d8a7051 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/GridStore.java @@ -0,0 +1,79 @@ +package com.aliyun.tablestore.grid; + +import com.alicloud.openservices.tablestore.model.TableOptions; +import com.alicloud.openservices.tablestore.model.search.IndexSchema; +import com.alicloud.openservices.tablestore.model.search.query.Query; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; +import com.aliyun.tablestore.grid.model.QueryParams; + +public interface GridStore { + + /** + * 创建相关的meta、data表，数据录入前调用。 + * @throws Exception + */ + void createStore() throws Exception; + + void createStore(TableOptions tableOptions) throws Exception; + + /** + * 写入gridDataSet的meta信息。 + * @param meta + * @throws Exception + */ + void putDataSetMeta(GridDataSetMeta meta) throws Exception; + + /** + * 更新meta信息。 + * @param meta + * @throws Exception + */ + void updateDataSetMeta(GridDataSetMeta meta) throws Exception; + + /** + * 通过gridDataSetId获取meta。 + * @param dataSetId + * @return + * @throws Exception + */ + GridDataSetMeta getDataSetMeta(String dataSetId) throws Exception; + + /** + * 创建meta表的多元索引。 + * @param indexName + * @param indexSchema + * @throws Exception + */ + void createMetaIndex(String indexName, IndexSchema indexSchema) throws Exception; + + /** + * 通过多种查询条件来查询符合条件的数据集。 + * @param indexName 多元索引名。 + * @param query 查询条件，可以通过QueryBuilder构建。 + * @param queryParams 查询相关参数，包括offset、limit、sort等。 + * @return + * @throws Exception + */ + QueryGridDataSetResult queryDataSets(String indexName, Query query, QueryParams queryParams) throws Exception; + + /** + * 获取GridDataWriter用于写入数据。 + * @param meta + * @return + */ + GridDataWriter getDataWriter(GridDataSetMeta meta); + + /** + * 获取GridDataFetcher用于读取数据。 + * @param meta + * @return + */ + GridDataFetcher getDataFetcher(GridDataSetMeta meta); + + /** + * 释放资源。 + */ + void close(); + +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGrid.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGrid.java new file mode 100644 index 0000000..ff671e2 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGrid.java @@ -0,0 +1,191 @@ +package com.aliyun.tablestore.grid; + +import com.alicloud.openservices.tablestore.*; +import com.alicloud.openservices.tablestore.core.ErrorCode; +import com.alicloud.openservices.tablestore.model.GetRowResponse; +import com.alicloud.openservices.tablestore.model.Row; +import com.alicloud.openservices.tablestore.model.TableOptions; +import com.alicloud.openservices.tablestore.model.UpdateTableRequest; +import com.alicloud.openservices.tablestore.model.search.CreateSearchIndexRequest; +import com.alicloud.openservices.tablestore.model.search.IndexSchema; +import com.alicloud.openservices.tablestore.model.search.SearchResponse; +import com.alicloud.openservices.tablestore.model.search.query.Query; +import com.alicloud.openservices.tablestore.writer.WriterConfig; +import com.aliyun.tablestore.grid.core.TableStoreDataDeleter; +import com.aliyun.tablestore.grid.core.RequestBuilder; +import com.aliyun.tablestore.grid.core.RowParser; +import com.aliyun.tablestore.grid.core.TableStoreDataFetcher; +import com.aliyun.tablestore.grid.core.TableStoreDataWriter; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; +import com.aliyun.tablestore.grid.model.QueryParams; +import ucar.nc2.ft.point.standard.TableConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class TableStoreGrid implements GridStore { + + private final TableStoreGridConfig config; + private final AsyncClientInterface asyncClient; + private ExecutorService writerExecutor; + private TableStoreWriter writer; + private TableOptions tableOptions; + + public TableStoreGrid(TableStoreGridConfig config) { + this.config = config; + this.asyncClient = new AsyncClient(config.getTableStoreEndpoint(), + config.getAccessId(), config.getAccessKey(), + config.getTableStoreInstance()); + } + + @Override + public void createStore() throws Exception { + this.tableOptions = new TableOptions(-1, 1); + createStore(tableOptions); + } + @Override + public void createStore(TableOptions tableOptions) throws Exception { + // create meta table + try { + this.tableOptions = tableOptions; + this.asyncClient.createTable(RequestBuilder.buildCreateMetaTableRequest(config.getMetaTableName(), tableOptions), null).get(); + } catch (TableStoreException ex) { + if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) { + throw ex; + } + } + + // create buffer table + try { + tableOptions.setAllowUpdate(false); // 禁止update操作 + this.asyncClient.createTable(RequestBuilder.buildCreateDataTableRequest(config.getDataTableName(), tableOptions), null).get(); + } catch (TableStoreException ex) { + if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) { + throw ex; + } + } + } + + /** + * 更新表设置 + * @param tableOptionsForUpdate + * @throws Exception + */ + public void updateStoreOption(TableOptions tableOptionsForUpdate) throws Exception { + // create meta table + try { + UpdateTableRequest metaUpdate = new UpdateTableRequest(config.getMetaTableName()); + metaUpdate.setTableOptionsForUpdate(tableOptionsForUpdate); + this.asyncClient.updateTable(metaUpdate, null).get(); + } catch (TableStoreException ex) { + if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) { + throw ex; + } + } + + // create buffer table + try { + UpdateTableRequest dataUpdate = new UpdateTableRequest(config.getDataTableName()); + dataUpdate.setTableOptionsForUpdate(tableOptionsForUpdate); + this.asyncClient.updateTable(dataUpdate, null).get(); + } catch (TableStoreException ex) { + if (!ex.getErrorCode().equals(ErrorCode.OBJECT_ALREADY_EXIST)) { + throw ex; + } + } + } + + + @Override + public void putDataSetMeta(GridDataSetMeta meta) throws Exception { + this.asyncClient.putRow( + RequestBuilder.buildPutMetaRequest(config.getMetaTableName(), meta), null).get(); + } + + public void deleteDataSetMeta(String uniqueKey) throws Exception { + this.asyncClient.deleteRow( + RequestBuilder.buildDeleteMetaRequest(config.getMetaTableName(), uniqueKey), + null + ); + } + + @Override + public void updateDataSetMeta(GridDataSetMeta meta) throws Exception { + this.asyncClient.updateRow( + RequestBuilder.buildUpdateMetaRequest(config.getMetaTableName(), meta), null).get(); + } + + @Override + public GridDataSetMeta getDataSetMeta(String gridDataSetId) throws Exception { + GetRowResponse getRowResponse = this.asyncClient.getRow( + RequestBuilder.buildGetMetaRequest(config.getMetaTableName(), gridDataSetId), null).get(); + if (getRowResponse.getRow() == null) { + return null; + } + return RowParser.parseMetaFromRow(getRowResponse.getRow()); + } + + @Override + public void createMetaIndex(String indexName, IndexSchema indexSchema) throws Exception { + CreateSearchIndexRequest request = new CreateSearchIndexRequest(); + request.setTableName(config.getMetaTableName()); + request.setIndexName(indexName); + request.setIndexSchema(indexSchema); + request.setTimeToLive(tableOptions.getTimeToLive(), TimeUnit.SECONDS); + this.asyncClient.createSearchIndex(request, null).get(); + } + + @Override + public QueryGridDataSetResult queryDataSets(String indexName, Query query, QueryParams queryParams) throws Exception { + SearchResponse searchResponse = this.asyncClient.search( + RequestBuilder.buildSearchRequest(config.getMetaTableName(), indexName, query, queryParams), + null).get(); + List metaList = new ArrayList(); + for (Row row : searchResponse.getRows()) { + metaList.add(RowParser.parseMetaFromRow(row)); + } + QueryGridDataSetResult result = new QueryGridDataSetResult(); + result.setGridDataSetMetas(metaList); + result.setAllSuccess(searchResponse.isAllSuccess()); + result.setNextToken(searchResponse.getNextToken()); + result.setTotalCount(searchResponse.getTotalCount()); + return result; + } + + @Override + public GridDataWriter getDataWriter(GridDataSetMeta meta) { + if (writer == null) { + synchronized (this) { + if (writer == null) { + this.writerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + this.writer = new DefaultTableStoreWriter(this.asyncClient, config.getDataTableName(), new WriterConfig(), null, this.writerExecutor); + } + } + } + return new TableStoreDataWriter(writer, config.getDataTableName(), meta); + } + + @Override + public GridDataFetcher getDataFetcher(GridDataSetMeta meta) { + return new TableStoreDataFetcher(asyncClient, config.getDataTableName(), meta, config.getDataSizeLimitForFetch()); + } + + public GridDataDeleter getDataDeleter(GridDataSetMeta meta) { + return new TableStoreDataDeleter(asyncClient, config.getDataTableName(), meta); + } + @Override + public synchronized void close() { + if (writer != null) { + this.writer.close(); + this.writerExecutor.shutdown(); + } + this.asyncClient.shutdown(); + } + + +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGridConfig.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGridConfig.java new file mode 100644 index 0000000..2958ddf --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/TableStoreGridConfig.java @@ -0,0 +1,70 @@ +package com.aliyun.tablestore.grid; + +public class TableStoreGridConfig { + + private String tableStoreEndpoint; + private String accessId; + private String accessKey; + private String tableStoreInstance; + + private String metaTableName; + private String dataTableName; + + private long dataSizeLimitForFetch = 20 * 1024 * 1024; + + public String getTableStoreEndpoint() { + return tableStoreEndpoint; + } + + public void setTableStoreEndpoint(String tableStoreEndpoint) { + this.tableStoreEndpoint = tableStoreEndpoint; + } + + public String getAccessId() { + return accessId; + } + + public void setAccessId(String accessId) { + this.accessId = accessId; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getTableStoreInstance() { + return tableStoreInstance; + } + + public void setTableStoreInstance(String tableStoreInstance) { + this.tableStoreInstance = tableStoreInstance; + } + + public String getMetaTableName() { + return metaTableName; + } + + public void setMetaTableName(String metaTableName) { + this.metaTableName = metaTableName; + } + + public String getDataTableName() { + return dataTableName; + } + + public void setDataTableName(String dataTableName) { + this.dataTableName = dataTableName; + } + + public long getDataSizeLimitForFetch() { + return dataSizeLimitForFetch; + } + + public void setDataSizeLimitForFetch(long dataSizeLimitForFetch) { + this.dataSizeLimitForFetch = dataSizeLimitForFetch; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/AttributionEnum.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/AttributionEnum.java new file mode 100644 index 0000000..7528b89 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/AttributionEnum.java @@ -0,0 +1,29 @@ +package com.aliyun.tablestore.grid.consts; + +/** + * @Author : shiyi + * @Date : 2024/5/8 15:27 + * @Description : meta表属性枚举 + */ +public enum AttributionEnum { + // + STATUS("status", "导入状态"), + CREATE_TIME("create_time", "创建时间"), + REFERENCE_TIME("reference_time", "起报时间"), + ; + public String name; + + public String info; + + public String getName() { + return name; + } + + public String getInfo() { + return info; + } + AttributionEnum(String name, String info) { + this.name = name; + this.info = info; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/Constants.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/Constants.java new file mode 100644 index 0000000..0c7eabd --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/consts/Constants.java @@ -0,0 +1,26 @@ +package com.aliyun.tablestore.grid.consts; + + +public class Constants { + + public static String GRID_DATA_SET_ID_PK_NAME = "_id"; + public static String VARIABLE_PK_NAME = "_variable"; + public static String T_PK_NAME = "_t"; + public static String Z_PK_NAME = "_z"; + + public static String DATA_TYPE_COL_NAME = "_data_type"; + public static String VARIABLE_LIST_COL_NAME = "_variable_list"; + public static String T_SIZE_COL_NAME = "_t_size"; + public static String Z_SIZE_COL_NAME = "_z_size"; + public static String X_SIZE_COL_NAME = "_x_size"; + public static String Y_SIZE_COL_NAME = "_y_size"; + public static String STORE_TYPE_COL_NAME = "_store_type"; + public static String X_SPLIT_COUNT_COL_NAME = "_x_split_count"; + public static String Y_SPLIT_COUNT_COL_NAME = "_y_split_count"; + + public static String DATA_BLOCK_COL_NAME_PREFIX = "block_"; + public static String DATA_BLOCK_COL_NAME_FORMAT = "block_%s_%s"; + + public static String T_LIST_COL_NAME = "time_list"; // 1M + public static int MAX_REQUEST_SIZE = 2 * 1024 * 1024; +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/QueryBuilder.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/QueryBuilder.java new file mode 100644 index 0000000..e010b3e --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/QueryBuilder.java @@ -0,0 +1,134 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.model.ColumnValue; +import com.alicloud.openservices.tablestore.model.search.query.*; +import com.aliyun.tablestore.grid.utils.ValueUtil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class QueryBuilder { + + enum Operator { + AND, + OR + } + + private final Operator operator; + private List filterQueries; + private List shouldQueries; + + QueryBuilder(Operator operator) { + this.operator = operator; + switch (operator) { + case AND: { + this.filterQueries = new ArrayList(); + break; + } + case OR: { + this.shouldQueries = new ArrayList(); + break; + } + default: + throw new IllegalArgumentException(); + } + } + + public static QueryBuilder or() { + return new QueryBuilder(Operator.OR); + } + + public static QueryBuilder and() { + return new QueryBuilder(Operator.AND); + } + + public QueryBuilder equal(String columnName, Object... values) { + TermsQuery termsQuery = new TermsQuery(); + termsQuery.setFieldName(columnName); + List columnValues = new ArrayList(); + for (Object value : values) { + columnValues.add(ValueUtil.toColumnValue(value)); + } + termsQuery.setTerms(columnValues); + return query(termsQuery); + } + + public QueryBuilder notEqual(String columnName, Object value) { + TermQuery termQuery = new TermQuery(); + termQuery.setFieldName(columnName); + termQuery.setTerm(ValueUtil.toColumnValue(value)); + BoolQuery boolQuery = new BoolQuery(); + boolQuery.setMustNotQueries(Collections.singletonList(termQuery)); + return query(boolQuery); + } + + public QueryBuilder greaterThan(String columnName, Object value) { + RangeQuery rangeQuery = new RangeQuery(); + rangeQuery.setFieldName(columnName); + rangeQuery.greaterThan(ValueUtil.toColumnValue(value)); + return query(rangeQuery); + } + + public QueryBuilder greaterThanEqual(String columnName, Object value) { + RangeQuery rangeQuery = new RangeQuery(); + rangeQuery.setFieldName(columnName); + rangeQuery.greaterThanOrEqual(ValueUtil.toColumnValue(value)); + return query(rangeQuery); + } + + public QueryBuilder lessThan(String columnName, Object value) { + RangeQuery rangeQuery = new RangeQuery(); + rangeQuery.setFieldName(columnName); + rangeQuery.lessThan(ValueUtil.toColumnValue(value)); + return query(rangeQuery); + } + + public QueryBuilder lessThanEqual(String columnName, Object value) { + RangeQuery rangeQuery = new RangeQuery(); + rangeQuery.setFieldName(columnName); + rangeQuery.lessThanOrEqual(ValueUtil.toColumnValue(value)); + return query(rangeQuery); + } + + public QueryBuilder prefix(String columnName, String prefix) { + PrefixQuery prefixQuery = new PrefixQuery(); + prefixQuery.setFieldName(columnName); + prefixQuery.setPrefix(prefix); + return query(prefixQuery); + } + + public QueryBuilder query(Query query) { + switch (operator) { + case AND: { + this.filterQueries.add(query); + break; + } + case OR: { + this.shouldQueries.add(query); + break; + } + default: + throw new IllegalArgumentException(); + } + return this; + } + + public Query build() { + switch (operator) { + case AND: { + BoolQuery boolQuery = new BoolQuery(); + boolQuery.setFilterQueries(filterQueries); + return boolQuery; + } + case OR: { + BoolQuery boolQuery = new BoolQuery(); + boolQuery.setShouldQueries(shouldQueries); + return boolQuery; + } + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RequestBuilder.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RequestBuilder.java new file mode 100644 index 0000000..4d58152 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RequestBuilder.java @@ -0,0 +1,149 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.core.utils.StringUtils; +import com.alicloud.openservices.tablestore.model.*; +import com.alicloud.openservices.tablestore.model.search.SearchQuery; +import com.alicloud.openservices.tablestore.model.search.SearchRequest; +import com.alicloud.openservices.tablestore.model.search.query.Query; +import com.aliyun.tablestore.grid.model.*; +import com.aliyun.tablestore.grid.utils.ValueUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.aliyun.tablestore.grid.consts.Constants.*; + +public class RequestBuilder { + + public static CreateTableRequest buildCreateMetaTableRequest(String tableName, TableOptions tableOptions) { + TableMeta meta = new TableMeta(tableName); + meta.addPrimaryKeyColumn(new PrimaryKeySchema(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyType.STRING)); + return new CreateTableRequest(meta, tableOptions); + } + + public static CreateTableRequest buildCreateDataTableRequest(String tableName, TableOptions tableOptions) { + TableMeta meta = new TableMeta(tableName); + meta.addPrimaryKeyColumn(new PrimaryKeySchema(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyType.STRING)); + meta.addPrimaryKeyColumn(new PrimaryKeySchema(VARIABLE_PK_NAME, PrimaryKeyType.STRING)); + meta.addPrimaryKeyColumn(new PrimaryKeySchema(T_PK_NAME, PrimaryKeyType.INTEGER)); + meta.addPrimaryKeyColumn(new PrimaryKeySchema(Z_PK_NAME, PrimaryKeyType.INTEGER)); + return new CreateTableRequest(meta, tableOptions); + } + + private static List buildMetaColumns(GridDataSetMeta meta) { + List columns = new ArrayList(); + columns.add(new Column(DATA_TYPE_COL_NAME, ColumnValue.fromString(meta.getDataType().toString()))); + columns.add(new Column(VARIABLE_LIST_COL_NAME, ColumnValue.fromString(StringUtils.join(",", meta.getVariables())))); + columns.add(new Column(T_SIZE_COL_NAME, ColumnValue.fromLong(meta.gettSize()))); + columns.add(new Column(Z_SIZE_COL_NAME, ColumnValue.fromLong(meta.getzSize()))); + columns.add(new Column(X_SIZE_COL_NAME, ColumnValue.fromLong(meta.getxSize()))); + columns.add(new Column(Y_SIZE_COL_NAME, ColumnValue.fromLong(meta.getySize()))); + columns.add(new Column(STORE_TYPE_COL_NAME, ColumnValue.fromString(meta.getStoreOptions().getStoreType().name()))); + if (meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) { + columns.add(new Column(X_SPLIT_COUNT_COL_NAME, ColumnValue.fromLong(meta.getStoreOptions().getxSplitCount()))); + columns.add(new Column(Y_SPLIT_COUNT_COL_NAME, ColumnValue.fromLong(meta.getStoreOptions().getySplitCount()))); + } + if (meta.getAttributes() != null) { + for (Map.Entry entry : meta.getAttributes().entrySet()) { + columns.add(new Column(entry.getKey(), ValueUtil.toColumnValue(entry.getValue()))); + } + } + if (meta.getForecastHours() != null) { + columns.add(new Column(T_LIST_COL_NAME, ColumnValue.fromString(StringUtils.join(",", meta.getForecastHours())))); + } + return columns; + } + + public static PutRowRequest buildPutMetaRequest(String metaTableName, GridDataSetMeta meta) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId())); + PrimaryKey pk = builder.build(); + RowPutChange rowPutChange = new RowPutChange(metaTableName, pk); + rowPutChange.addColumns(buildMetaColumns(meta)); + return new PutRowRequest(rowPutChange); + } + + public static UpdateRowRequest buildUpdateMetaRequest(String metaTableName, GridDataSetMeta meta) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId())); + PrimaryKey pk = builder.build(); + RowUpdateChange rowUpdateChange = new RowUpdateChange(metaTableName, pk); + rowUpdateChange.put(buildMetaColumns(meta)); + return new UpdateRowRequest(rowUpdateChange); + } + + public static GetRowRequest buildGetMetaRequest(String metaTableName, String uniqueKey) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(uniqueKey)); + PrimaryKey pk = builder.build(); + SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(metaTableName); + criteria.setMaxVersions(1); + criteria.setPrimaryKey(pk); + GetRowRequest getRowRequest = new GetRowRequest(); + getRowRequest.setRowQueryCriteria(criteria); + return getRowRequest; + } + + public static SearchRequest buildSearchRequest(String metaTableName, String indexName, Query query, QueryParams params) { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.setQuery(query); + if (params.getOffset() != null) { + searchQuery.setOffset(params.getOffset()); + } + if (params.getLimit() != null) { + searchQuery.setLimit(params.getLimit()); + } + if (params.getSort() != null) { + searchQuery.setSort(params.getSort()); + } + if (params.getToken() != null) { + searchQuery.setToken(params.getToken()); + } + searchQuery.setGetTotalCount(params.isGetTotalCount()); + SearchRequest searchRequest = new SearchRequest(metaTableName, indexName, searchQuery); + SearchRequest.ColumnsToGet columnsToGet = new SearchRequest.ColumnsToGet(); + columnsToGet.setReturnAll(true); + searchRequest.setColumnsToGet(columnsToGet); + return searchRequest; + } + + public static GetRowRequest buildGetDataRequest(GetDataParam getDataParam) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(getDataParam.getDataSetId())); + builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(getDataParam.getVariable())); + builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(getDataParam.getT())); + builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(getDataParam.getZ())); + PrimaryKey pk = builder.build(); + + SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(getDataParam.getDataTableName()); + criteria.setMaxVersions(1); + criteria.setPrimaryKey(pk); + if (getDataParam.getColumnsToGet() != null) { + criteria.addColumnsToGet(getDataParam.getColumnsToGet()); + } + return new GetRowRequest(criteria); + } + + public static DeleteRowRequest buildDeleteDataRequest(DeleteDataParam deleteDataParam) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(deleteDataParam.getDataSetId())); + builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(deleteDataParam.getVariable())); + builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(deleteDataParam.getT())); + builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(deleteDataParam.getZ())); + PrimaryKey pk = builder.build(); + + RowDeleteChange rowDeleteChange = new RowDeleteChange(deleteDataParam.getDataTableName(), pk); + return new DeleteRowRequest(rowDeleteChange); + } + public static DeleteRowRequest buildDeleteMetaRequest(String metaTableName, String uniqueKey) { + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(uniqueKey)); + PrimaryKey pk = builder.build(); + SingleRowQueryCriteria criteria = new SingleRowQueryCriteria(metaTableName); + criteria.setMaxVersions(1); + criteria.setPrimaryKey(pk); + RowDeleteChange rowDeleteChange = new RowDeleteChange(metaTableName, pk); + return new DeleteRowRequest(rowDeleteChange); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RowParser.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RowParser.java new file mode 100644 index 0000000..3fdd77e --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/RowParser.java @@ -0,0 +1,75 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.model.Column; +import com.alicloud.openservices.tablestore.model.Row; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.StoreOptions; +import com.aliyun.tablestore.grid.model.grid.Grid2D; +import com.aliyun.tablestore.grid.model.grid.Plane; +import com.aliyun.tablestore.grid.model.grid.Range; +import com.aliyun.tablestore.grid.utils.BlockUtil; +import com.aliyun.tablestore.grid.utils.ValueUtil; +import ucar.ma2.DataType; + +import java.nio.ByteBuffer; +import java.util.*; + +import static com.aliyun.tablestore.grid.consts.Constants.*; + +public class RowParser { + + public static GridDataSetMeta parseMetaFromRow(Row row) { + String uniqueKey = row.getPrimaryKey().getPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME).getValue().asString(); + DataType dataType = DataType.getType(row.getColumn(DATA_TYPE_COL_NAME).get(0).getValue().asString()); + List variables = Arrays.asList(row.getColumn(VARIABLE_LIST_COL_NAME).get(0).getValue().asString().split(",")); + int tSize = (int) row.getColumn(T_SIZE_COL_NAME).get(0).getValue().asLong(); + int zSize = (int) row.getColumn(Z_SIZE_COL_NAME).get(0).getValue().asLong(); + int xSize = (int) row.getColumn(X_SIZE_COL_NAME).get(0).getValue().asLong(); + int ySize = (int) row.getColumn(Y_SIZE_COL_NAME).get(0).getValue().asLong(); + + List refTimes = Arrays.asList(row.getColumn(T_LIST_COL_NAME).get(0).getValue().asString().split(",")); + + StoreOptions.StoreType storeType = StoreOptions.StoreType.valueOf( + row.getColumn(STORE_TYPE_COL_NAME).get(0).getValue().asString()); + StoreOptions storeOptions = new StoreOptions(storeType); + if (storeType.equals(StoreOptions.StoreType.SLICE)) { + storeOptions.setxSplitCount((int) row.getColumn(X_SPLIT_COUNT_COL_NAME).get(0).getValue().asLong()); + storeOptions.setySplitCount((int) row.getColumn(Y_SPLIT_COUNT_COL_NAME).get(0).getValue().asLong()); + } + Map attributes = new HashMap(); + for (Column column : row.getColumns()) { + if (!column.getName().startsWith("_")) { + attributes.put(column.getName(), ValueUtil.toObject(column.getValue())); + } + } + GridDataSetMeta meta = new GridDataSetMeta(uniqueKey, dataType, variables, tSize, zSize, xSize, ySize, storeOptions); + meta.setForecastHours(refTimes); + meta.setAttributes(attributes); + return meta; + } + + public static Grid2D parseGridFromRow(Row row, Plane plane, GridDataSetMeta meta, byte[] buffer, int pos) { + if (!meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) { + throw new IllegalArgumentException("unsupported store type"); + } + int blockXSize = (meta.getxSize() - 1) / meta.getStoreOptions().getxSplitCount() + 1; + int blockYSize = (meta.getySize() - 1) / meta.getStoreOptions().getySplitCount() + 1; + List blocks = new ArrayList(); + for (Column column : row.getColumns()) { + if (column.getName().startsWith(DATA_BLOCK_COL_NAME_PREFIX)) { + String[] strs = column.getName().split("_"); + int xStart = Integer.valueOf(strs[strs.length - 2]); + int yStart = Integer.valueOf(strs[strs.length - 1]); + Range xRange = new Range(xStart, Math.min(xStart + blockXSize, meta.getxSize())); + Range yRange = new Range(yStart, Math.min(yStart + blockYSize, meta.getySize())); + int[] origin = new int[] {xStart, yStart}; + int[] shape = new int[] {xRange.getSize(), yRange.getSize()}; + Grid2D grid2D = new Grid2D(ByteBuffer.wrap(column.getValue().asBinary()), meta.getDataType(), origin, shape); + blocks.add(grid2D); + } + } + Grid2D grid2D = BlockUtil.buildGrid2DFromBlocks(plane, meta.getDataType(), blocks, buffer, pos); + return grid2D; + } + +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataDeleter.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataDeleter.java new file mode 100644 index 0000000..9bf11a1 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataDeleter.java @@ -0,0 +1,68 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.AsyncClientInterface; +import com.alicloud.openservices.tablestore.model.BatchWriteRowRequest; +import com.alicloud.openservices.tablestore.model.BatchWriteRowResponse; +import com.aliyun.tablestore.grid.GridDataDeleter; +import com.aliyun.tablestore.grid.model.DeleteDataParam; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +/** + * @Author : shiyi + * @Date : 2024/2/3 11:51 + * @Description : + */ +@Slf4j +public class TableStoreDataDeleter implements GridDataDeleter { + private final AsyncClientInterface asyncClient; + private final GridDataSetMeta meta; + private final String dataTableName; + private final Collection variables; + + public TableStoreDataDeleter(AsyncClientInterface asyncClient, String dataTableName, GridDataSetMeta meta) { + this.asyncClient = asyncClient; + this.dataTableName = dataTableName; + this.meta = meta; + this.variables = this.meta.getVariables(); + } + + + @Override + public void delete(String dataSetId) throws ExecutionException, InterruptedException { + + Set refTimes = meta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet()); + for (String variable : meta.getVariables()) { + for (int t = 0; t < meta.gettSize(); t++) { + deleteByTime(dataSetId, variable, t); + } + } + } + + private void deleteByTime(String dataSetId, String variable, int t) throws InterruptedException, ExecutionException { + BatchWriteRowRequest batchWriteRowRequest = new BatchWriteRowRequest(); + for (int z = 0; z < meta.getzSize(); z++) { + DeleteDataParam param = new DeleteDataParam(dataTableName, dataSetId, variable, t, z); + // 添加到batch操作中, 如果该行不存在，也不会报错 + batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange()); + } + BatchWriteRowResponse response = asyncClient.batchWriteRow(batchWriteRowRequest, null).get(); + log.info("variable={}, t={} 是否删除成功：{}", variable, t, response.isAllSucceed()); + if (!response.isAllSucceed()) { + for (BatchWriteRowResponse.RowResult rowResult : response.getFailedRows()) { + log.warn("失败的行：{}", batchWriteRowRequest.getRowChange(rowResult.getTableName(), rowResult.getIndex()).getPrimaryKey()); + log.warn("失败原因：{}", rowResult.getError()); + } + /** + * 可以通过createRequestForRetry方法再构造一个请求对失败的行进行重试。此处只给出构造重试请求的部分。 + * 推荐的重试方法是使用SDK的自定义重试策略功能，支持对batch操作的部分行错误进行重试。设置重试策略后，调用接口处无需增加重试代码。 + */ + BatchWriteRowRequest retryRequest = batchWriteRowRequest.createRequestForRetry(response.getFailedRows()); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataFetcher.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataFetcher.java new file mode 100644 index 0000000..ae0d53e --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataFetcher.java @@ -0,0 +1,224 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.AsyncClientInterface; +import com.alicloud.openservices.tablestore.TableStoreCallback; +import com.alicloud.openservices.tablestore.model.GetRowRequest; +import com.alicloud.openservices.tablestore.model.GetRowResponse; +import com.aliyun.tablestore.grid.GridDataFetcher; +import com.aliyun.tablestore.grid.model.GetDataParam; +import com.aliyun.tablestore.grid.model.GridDataSet; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.StoreOptions; +import com.aliyun.tablestore.grid.model.grid.*; +import com.aliyun.tablestore.grid.utils.BlockUtil; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.aliyun.tablestore.grid.consts.Constants.DATA_BLOCK_COL_NAME_FORMAT; + +/** + * not thread-safe + */ +public class TableStoreDataFetcher implements GridDataFetcher { + + private final AsyncClientInterface asyncClient; + private final GridDataSetMeta meta; + private final String tableName; + private final long dataSizeLimitForFetch; + private Collection variables; + private Range tRange; + private Range zRange; + private Range xRange; + private Range yRange; + + public TableStoreDataFetcher(AsyncClientInterface asyncClient, String tableName, GridDataSetMeta meta, long dataSizeLimitForFetch) { + this.asyncClient = asyncClient; + this.tableName = tableName; + this.meta = meta; + this.dataSizeLimitForFetch = dataSizeLimitForFetch; + this.variables = this.meta.getVariables(); + this.tRange = new Range(0, this.meta.gettSize()); + this.zRange = new Range(0, this.meta.getzSize()); + this.xRange = new Range(0, this.meta.getxSize()); + this.yRange = new Range(0, this.meta.getySize()); + } + + @Override + public GridDataFetcher setVariablesToGet(Collection variables) { + this.variables = variables; + return this; + } + + @Override + public GridDataFetcher setT(int t) { + return setTRange(new Range(t, t+1)); + } + + @Override + public GridDataFetcher setTRange(Range range) { + if (range.getStart() < 0 || range.getEnd() > meta.gettSize()) { + throw new IllegalArgumentException("range invalid"); + } + this.tRange = range; + return this; + } + + @Override + public GridDataFetcher setZ(int z) { + return setZRange(new Range(z, z+1)); + } + + @Override + public GridDataFetcher setZRange(Range range) { + if (range.getStart() < 0 || range.getEnd() > meta.getzSize()) { + throw new IllegalArgumentException("range invalid"); + } + this.zRange = range; + return this; + } + + @Override + public GridDataFetcher setX(int x) { + return setXRange(new Range(x, x+1)); + } + + @Override + public GridDataFetcher setXRange(Range range) { + if (range.getStart() < 0 || range.getEnd() > meta.getxSize()) { + throw new IllegalArgumentException("range invalid"); + } + this.xRange = range; + return this; + } + + @Override + public GridDataFetcher setY(int y) { + return setYRange(new Range(y, y+1)); + } + + @Override + public GridDataFetcher setYRange(Range range) { + if (range.getStart() < 0 || range.getEnd() > meta.getySize()) { + throw new IllegalArgumentException("range invalid"); + } + this.yRange = range; + return this; + } + + @Override + public GridDataFetcher setOriginShape(int[] origin, int[] shape) { + if (origin.length != 4 || shape.length != 4) { + throw new IllegalArgumentException("the length of origin and shape must be 4"); + } + setTRange(new Range(origin[0], origin[0] + shape[0])); + setZRange(new Range(origin[1], origin[1] + shape[1])); + setXRange(new Range(origin[2], origin[2] + shape[2])); + setYRange(new Range(origin[3], origin[3] + shape[3])); + return this; + } + + public int[] getOrigin() { + return new int[] {tRange.getStart(), zRange.getStart(), xRange.getStart(), yRange.getStart()}; + } + + public int[] getShape() { + return new int[] {tRange.getSize(), zRange.getSize(), xRange.getSize(), yRange.getSize()}; + } + + private long calcDataSize(int variableCount) { + long dataSize = variableCount; + return dataSize * meta.getDataType().getSize() * tRange.getSize() * zRange.getSize() * xRange.getSize() * yRange.getSize(); + } + + private List getColumnsToGet() { + if (!meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) { + throw new IllegalArgumentException("unsupported store type"); + } + Plane plane = new Plane(new Range(meta.getxSize()), new Range(meta.getySize())); + Plane subPlane = new Plane(xRange, yRange); + if (plane.equals(subPlane)) { + return null; + } + List columnsToGet = new ArrayList(); + List points = BlockUtil.calcBlockPointsCanCoverSubPlane(plane, subPlane, + meta.getStoreOptions().getxSplitCount(), meta.getStoreOptions().getySplitCount()); + for (Point point : points) { + columnsToGet.add(String.format(DATA_BLOCK_COL_NAME_FORMAT, point.getX(), point.getY())); + } + return columnsToGet; + } + + private void addTask(final AtomicInteger counter, final byte[] buffer, final int pos, String variable, int t, int z, final CountDownLatch latch, final Queue exceptions) { + GetDataParam param = new GetDataParam(tableName, meta.getGridDataSetId(), variable, t, z, getColumnsToGet()); + asyncClient.getRow(RequestBuilder.buildGetDataRequest(param), new TableStoreCallback() { + @Override + public void onCompleted(GetRowRequest req, GetRowResponse res) { + try { + if (res.getRow() == null) { + exceptions.add(new RuntimeException("the row in not exist, pk: " + req.getRowQueryCriteria().getPrimaryKey())); + } + RowParser.parseGridFromRow(res.getRow(), new Plane(xRange, yRange), meta, buffer, pos); + counter.incrementAndGet(); + } catch (Exception ex) { + exceptions.add(ex); + } finally { + latch.countDown(); + } + } + + @Override + public void onFailed(GetRowRequest req, Exception ex) { + try { + exceptions.add(ex); + } finally { + latch.countDown(); + } + } + }); + } + + @Override + public GridDataSet fetch() throws Exception { + long totalFetchDataSize = calcDataSize(variables.size()); + if (totalFetchDataSize == 0) { + throw new RuntimeException("no data to fetch"); + } + if (totalFetchDataSize > dataSizeLimitForFetch) { + throw new RuntimeException("exceed the max data limit for fetch"); + } + GridDataSet dataSet = new GridDataSet(meta); + CountDownLatch latch = new CountDownLatch(variables.size() * tRange.getSize() * zRange.getSize()); + Queue exceptions = new ConcurrentLinkedQueue(); + AtomicInteger counter = new AtomicInteger(); + int taskCount = 0; + for (String variable : variables) { + int dataSize = (int) calcDataSize(1); + byte[] data = new byte[dataSize]; + ByteBuffer buffer = ByteBuffer.wrap(data).asReadOnlyBuffer(); + dataSet.addVariable(variable, new Grid4D(buffer, meta.getDataType(), getOrigin(), getShape())); + int curPos = 0; + for (int t = tRange.getStart(); t < tRange.getEnd(); t++) { + for (int z = zRange.getStart(); z < zRange.getEnd(); z++) { + addTask(counter, data, curPos, variable, t, z, latch, exceptions); + curPos += xRange.getSize() * yRange.getSize() * meta.getDataType().getSize(); + taskCount++; + } + } + } + latch.await(); + if (!exceptions.isEmpty()) { + throw exceptions.peek(); + } + if (counter.get() != taskCount) { + throw new RuntimeException("not all task success"); + } + return dataSet; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataWriter.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataWriter.java new file mode 100644 index 0000000..b795a35 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/core/TableStoreDataWriter.java @@ -0,0 +1,98 @@ +package com.aliyun.tablestore.grid.core; + +import com.alicloud.openservices.tablestore.TableStoreWriter; +import com.alicloud.openservices.tablestore.model.*; +import com.aliyun.tablestore.grid.GridDataWriter; +import com.aliyun.tablestore.grid.model.grid.Grid2D; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.StoreOptions; +import com.aliyun.tablestore.grid.model.grid.Plane; +import com.aliyun.tablestore.grid.utils.BlockUtil; +import ucar.ma2.InvalidRangeException; + +import java.util.ArrayList; +import java.util.List; + +import static com.aliyun.tablestore.grid.consts.Constants.*; + +public class TableStoreDataWriter implements GridDataWriter { + + private final String tableName; + private final GridDataSetMeta meta; + private final TableStoreWriter writer; + + public TableStoreDataWriter(TableStoreWriter writer, String tableName, GridDataSetMeta dataSetMeta) { + this.writer = writer; + this.tableName = tableName; + this.meta = dataSetMeta; + } + + private void checkDataSize(String variable, int t, int z, Grid2D grid2D) { + if (!meta.getVariables().contains(variable)) { + throw new IllegalArgumentException("The data set dose not include this variable: " + variable); + } + if (t >= meta.gettSize()) { + throw new IllegalArgumentException("t must be in range: [0, " + meta.gettSize() + ")"); + } + if (z >= meta.getzSize()) { + throw new IllegalArgumentException("z must be in range: [0, " + meta.getzSize() + ")"); + } + Plane plane = new Plane(grid2D.getOrigin(), grid2D.getShape()); + if (plane.getxRange().getStart() != 0 || plane.getyRange().getStart() != 0) { + throw new IllegalArgumentException("xStart and yStart in grid2D must be 0"); + } + if (plane.getxRange().getSize() != meta.getxSize()) { + throw new IllegalArgumentException("xSize in grid2D must be equal to gridDataSetMeta's xSize"); + } + if (plane.getyRange().getSize() != meta.getySize()) { + throw new IllegalArgumentException("ySize in grid2D must be equal to gridDataSetMeta's ySize"); + } + } + + private List splitDataToColumns(Grid2D grid2D) throws InvalidRangeException { + List columns = new ArrayList(); + List blocks = BlockUtil.splitGrid2DToBlocks(grid2D, meta.getStoreOptions().getxSplitCount(), meta.getStoreOptions().getySplitCount()); + for (Grid2D block : blocks) { + columns.add(new Column(String.format(DATA_BLOCK_COL_NAME_FORMAT, block.getPlane().getxRange().getStart(), + block.getPlane().getyRange().getStart()), ColumnValue.fromBinary(block.getDataAsByteArray()))); + } + return columns; + } + + private void writeToTableStore(String variable, int t, int z, List columns) { + if (columns.size() == 0) { + throw new IllegalArgumentException("columns are empty"); + } + PrimaryKeyBuilder builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + builder.addPrimaryKeyColumn(GRID_DATA_SET_ID_PK_NAME, PrimaryKeyValue.fromString(meta.getGridDataSetId())); + builder.addPrimaryKeyColumn(VARIABLE_PK_NAME, PrimaryKeyValue.fromString(variable)); + builder.addPrimaryKeyColumn(T_PK_NAME, PrimaryKeyValue.fromLong(t)); + builder.addPrimaryKeyColumn(Z_PK_NAME, PrimaryKeyValue.fromLong(z)); + PrimaryKey pk = builder.build(); + + RowUpdateChange rowUpdateChange = new RowUpdateChange(tableName, pk); + int currentSize = 0; + for (int i = 0; i < columns.size(); i++) { + if (currentSize != 0 && (currentSize + columns.get(i).getDataSize()) > MAX_REQUEST_SIZE) { + writer.addRowChange(rowUpdateChange); + rowUpdateChange = new RowUpdateChange(tableName, pk); + currentSize = 0; + } + rowUpdateChange.put(columns.get(i)); + currentSize += columns.get(i).getDataSize(); + } + writer.addRowChange(rowUpdateChange); + writer.flush(); + } + + @Override + public void writeGrid2D(String variable, int t, int z, Grid2D grid2D) throws Exception { + checkDataSize(variable, t, z, grid2D); + if (meta.getStoreOptions().getStoreType().equals(StoreOptions.StoreType.SLICE)) { + List columns = splitDataToColumns(grid2D); + writeToTableStore(variable, t, z, columns); + } else { + throw new IllegalArgumentException("unsupported store type"); + } + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/DeleteDataParam.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/DeleteDataParam.java new file mode 100644 index 0000000..2c0b374 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/DeleteDataParam.java @@ -0,0 +1,20 @@ +package com.aliyun.tablestore.grid.model; + +import lombok.Data; + +@Data +public class DeleteDataParam { + private String dataTableName; + private String dataSetId; + private String variable; + private int t; + private int z; + + public DeleteDataParam(String dataTableName, String dataSetId, String variable, int t, int z) { + this.dataTableName = dataTableName; + this.dataSetId = dataSetId; + this.variable = variable; + this.t = t; + this.z = z; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GetDataParam.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GetDataParam.java new file mode 100644 index 0000000..8941a73 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GetDataParam.java @@ -0,0 +1,70 @@ +package com.aliyun.tablestore.grid.model; + +import java.util.List; + +public class GetDataParam { + + private String dataTableName; + private String dataSetId; + private String variable; + private int t; + private int z; + private List columnsToGet; + + public GetDataParam(String dataTableName, String dataSetId, String variable, int t, int z, List columnsToGet) { + this.dataTableName = dataTableName; + this.dataSetId = dataSetId; + this.variable = variable; + this.t = t; + this.z = z; + this.columnsToGet = columnsToGet; + } + + public String getDataTableName() { + return dataTableName; + } + + public void setDataTableName(String dataTableName) { + this.dataTableName = dataTableName; + } + + public String getDataSetId() { + return dataSetId; + } + + public void setDataSetId(String dataSetId) { + this.dataSetId = dataSetId; + } + + public String getVariable() { + return variable; + } + + public void setVariable(String variable) { + this.variable = variable; + } + + public int getT() { + return t; + } + + public void setT(int t) { + this.t = t; + } + + public int getZ() { + return z; + } + + public void setZ(int z) { + this.z = z; + } + + public List getColumnsToGet() { + return columnsToGet; + } + + public void setColumnsToGet(List columnsToGet) { + this.columnsToGet = columnsToGet; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSet.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSet.java new file mode 100644 index 0000000..3c843ec --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSet.java @@ -0,0 +1,42 @@ +package com.aliyun.tablestore.grid.model; + +import com.aliyun.tablestore.grid.model.grid.Grid4D; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class GridDataSet { + + private final GridDataSetMeta meta; + + private final Map variables; + + public GridDataSet(GridDataSetMeta meta) { + this.meta = meta; + this.variables = new ConcurrentHashMap(); + } + + public GridDataSet(GridDataSetMeta meta, Map variables) { + this.meta = meta; + this.variables = variables; + } + + public void addVariable(String variable, Grid4D grid4D) { + this.variables.put(variable, grid4D); + } + + public Grid4D getVariable(String variable) { + if (variables == null) { + return null; + } + return variables.get(variable); + } + + public Map getVariables() { + return variables; + } + + public GridDataSetMeta getMeta() { + return meta; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSetMeta.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSetMeta.java new file mode 100644 index 0000000..2852b0c --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/GridDataSetMeta.java @@ -0,0 +1,124 @@ +package com.aliyun.tablestore.grid.model; + +import ucar.ma2.DataType; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class GridDataSetMeta { + + private String gridDataSetId; + private final DataType dataType; + private List variables; + private int tSize; + private int zSize; + private int xSize; + private int ySize; + private List forecastHours; + private final StoreOptions storeOptions; + private Map attributes; + + public GridDataSetMeta(String gridDataSetId, DataType dataType, List variables, int tSize, int zSize, int xSize, int ySize, StoreOptions storeOptions) { + assert gridDataSetId != null; + assert variables != null; + assert storeOptions != null; + + this.gridDataSetId = gridDataSetId; + this.dataType = dataType; + this.variables = variables; + this.tSize = tSize; + this.zSize = zSize; + this.xSize = xSize; + this.ySize = ySize; + this.storeOptions = storeOptions; + } + + public String getGridDataSetId() { + return gridDataSetId; + } + + public void setGridDataSetId(String gridDataSetId) { + this.gridDataSetId = gridDataSetId; + } + + public List getVariables() { + return variables; + } + + public void setVariables(List variables) { + this.variables = variables; + } + + public int gettSize() { + return tSize; + } + + public void settSize(int tSize) { + this.tSize = tSize; + } + + public int getzSize() { + return zSize; + } + + public void setzSize(int zSize) { + this.zSize = zSize; + } + + public int getxSize() { + return xSize; + } + + public void setxSize(int xSize) { + this.xSize = xSize; + } + + public int getySize() { + return ySize; + } + + public void setySize(int ySize) { + this.ySize = ySize; + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + assert attributes != null; + for (String key : attributes.keySet()) { + if (key.startsWith("_")) { + throw new IllegalArgumentException("attribute key can't start with \"_\""); + } + } + this.attributes = attributes; + } + + public void addAttribute(String key, Object value) { + if (key.startsWith("_")) { + throw new IllegalArgumentException("attribute key can't start with \"_\""); + } + if (this.attributes == null) { + this.attributes = new ConcurrentHashMap(); + } + this.attributes.put(key, value); + } + + public StoreOptions getStoreOptions() { + return storeOptions; + } + + public DataType getDataType() { + return dataType; + } + + public void setForecastHours(List forecastHours) { + this.forecastHours = forecastHours; + } + + public List getForecastHours() { + return forecastHours; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryGridDataSetResult.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryGridDataSetResult.java new file mode 100644 index 0000000..1d57aeb --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryGridDataSetResult.java @@ -0,0 +1,43 @@ +package com.aliyun.tablestore.grid.model; + +import java.util.List; + +public class QueryGridDataSetResult { + + private List gridDataSetMetas; + private long totalCount; + private boolean isAllSuccess; + private byte[] nextToken; + + public List getGridDataSetMetas() { + return gridDataSetMetas; + } + + public void setGridDataSetMetas(List gridDataSetMetas) { + this.gridDataSetMetas = gridDataSetMetas; + } + + public long getTotalCount() { + return totalCount; + } + + public void setTotalCount(long totalCount) { + this.totalCount = totalCount; + } + + public boolean isAllSuccess() { + return isAllSuccess; + } + + public void setAllSuccess(boolean allSuccess) { + isAllSuccess = allSuccess; + } + + public byte[] getNextToken() { + return nextToken; + } + + public void setNextToken(byte[] nextToken) { + this.nextToken = nextToken; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryParams.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryParams.java new file mode 100644 index 0000000..3f6ff84 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/QueryParams.java @@ -0,0 +1,82 @@ +package com.aliyun.tablestore.grid.model; + + +import com.alicloud.openservices.tablestore.model.search.sort.Sort; + +public class QueryParams { + + private Integer offset; + private Integer limit; + private Sort sort; + private byte[] token; + private boolean getTotalCount = false; + + public QueryParams() { + } + + public QueryParams(int limit) { + this.limit = limit; + } + + public QueryParams(int offset, int limit) { + this.offset = offset; + this.limit = limit; + } + + public QueryParams(int offset, int limit, Sort sort) { + this.offset = offset; + this.limit = limit; + this.sort = sort; + } + + public QueryParams(byte[] token, int limit) { + this.token = token; + this.limit = limit; + } + + + public Integer getOffset() { + return offset; + } + + public QueryParams setOffset(Integer offset) { + this.offset = offset; + return this; + } + + public Integer getLimit() { + return limit; + } + + public QueryParams setLimit(Integer limit) { + this.limit = limit; + return this; + } + + public Sort getSort() { + return sort; + } + + public QueryParams setSort(Sort sort) { + this.sort = sort; + return this; + } + + public byte[] getToken() { + return token; + } + + public QueryParams setToken(byte[] token) { + this.token = token; + return this; + } + + public boolean isGetTotalCount() { + return getTotalCount; + } + + public QueryParams setGetTotalCount(boolean getTotalCount) { + this.getTotalCount = getTotalCount; + return this; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/StoreOptions.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/StoreOptions.java new file mode 100644 index 0000000..71186c3 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/StoreOptions.java @@ -0,0 +1,41 @@ +package com.aliyun.tablestore.grid.model; + +public class StoreOptions { + + public enum StoreType { + SLICE + } + + private StoreType storeType; + private int xSplitCount = 10; + private int ySplitCount = 10; + + public StoreOptions(StoreType storeType) { + this.storeType = storeType; + } + + public StoreType getStoreType() { + return storeType; + } + + public void setStoreType(StoreType storeType) { + this.storeType = storeType; + } + + public int getxSplitCount() { + return xSplitCount; + } + + public void setxSplitCount(int xSplitCount) { + this.xSplitCount = xSplitCount; + } + + public int getySplitCount() { + return ySplitCount; + } + + public void setySplitCount(int ySplitCount) { + this.ySplitCount = ySplitCount; + } + +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid.java new file mode 100644 index 0000000..c4c79dd --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid.java @@ -0,0 +1,58 @@ +package com.aliyun.tablestore.grid.model.grid; + +import ucar.ma2.Array; +import ucar.ma2.DataType; + +import java.nio.ByteBuffer; + +public abstract class Grid { + + protected ByteBuffer buffer; + protected DataType dataType; + protected int[] origin; + protected int[] shape; + + public Grid(ByteBuffer buffer, DataType dataType, int[] origin, int[] shape) { + this.buffer = buffer; + this.dataType = dataType; + this.origin = origin; + this.shape = shape; + int size = dataType.getSize(); + for (int i = 0; i < shape.length; i++) { + size *= shape[i]; + } + if (buffer.remaining() != size) { + throw new IllegalArgumentException("data length and shape mismatch"); + } + if (origin.length != shape.length) { + throw new IllegalArgumentException("the length of origin and shape mismatch"); + } + } + + public int getDataSize() { + return buffer.remaining(); + } + + public byte[] getDataAsByteArray() { + byte[] data = new byte[getDataSize()]; + buffer.duplicate().get(data); + return data; + } + + public int[] getOrigin() { + return origin; + } + + public int[] getShape() { + return shape; + } + + public DataType getDataType() { + return dataType; + } + + public Array toArray() { + Array array = Array.factory(dataType, shape, buffer.duplicate()); + return array; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid2D.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid2D.java new file mode 100644 index 0000000..37a6dac --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid2D.java @@ -0,0 +1,19 @@ +package com.aliyun.tablestore.grid.model.grid; + +import ucar.ma2.DataType; + +import java.nio.ByteBuffer; + +public class Grid2D extends Grid { + + public Grid2D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) { + super(data, dataType, origin, shape); + if (origin.length != 2 || shape.length != 2) { + throw new IllegalArgumentException("the length of origin and shape must be 2"); + } + } + + public Plane getPlane() { + return new Plane(getOrigin(), getShape()); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid3D.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid3D.java new file mode 100644 index 0000000..b342912 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid3D.java @@ -0,0 +1,30 @@ +package com.aliyun.tablestore.grid.model.grid; + +import ucar.ma2.DataType; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class Grid3D extends Grid { + + public Grid3D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) { + super(data, dataType, origin, shape); + if (origin.length != 3 || shape.length != 3) { + throw new IllegalArgumentException("the length of origin and shape must be 2"); + } + } + + public Grid2D getGrid2D(int idx) { + if (idx < 0 || idx >= shape[0]) { + throw new IllegalArgumentException("index out of range"); + } + int itemSize = shape[1] * shape[2] * dataType.getSize(); + int pos = idx * itemSize; + ByteBuffer newBuffer = buffer.slice(); + newBuffer.position(pos); + newBuffer.limit(pos + itemSize); + newBuffer = newBuffer.slice(); + return new Grid2D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length), + Arrays.copyOfRange(shape, 1, shape.length)); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid4D.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid4D.java new file mode 100644 index 0000000..0cd97bb --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Grid4D.java @@ -0,0 +1,30 @@ +package com.aliyun.tablestore.grid.model.grid; + +import ucar.ma2.DataType; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class Grid4D extends Grid { + + public Grid4D(ByteBuffer data, DataType dataType, int[] origin, int[] shape) { + super(data, dataType, origin, shape); + if (origin.length != 4 || shape.length != 4) { + throw new IllegalArgumentException("the length of origin and shape must be 2"); + } + } + + public Grid3D getGrid3D(int idx) { + if (idx < 0 || idx >= shape[0]) { + throw new IllegalArgumentException("index out of range"); + } + int itemSize = shape[1] * shape[2] * shape[3] * dataType.getSize(); + int pos = idx * itemSize; + ByteBuffer newBuffer = buffer.slice(); + newBuffer.position(pos); + newBuffer.limit(pos + itemSize); + newBuffer = newBuffer.slice(); + return new Grid3D(newBuffer, dataType, Arrays.copyOfRange(origin, 1, origin.length), + Arrays.copyOfRange(shape, 1, shape.length)); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Plane.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Plane.java new file mode 100644 index 0000000..66e57f3 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Plane.java @@ -0,0 +1,55 @@ +package com.aliyun.tablestore.grid.model.grid; + +public class Plane { + + private Range xRange; + private Range yRange; + + public Plane(Range xRange, Range yRange) { + this.xRange = xRange; + this.yRange = yRange; + } + + public Plane(int[] origin, int[] shape) { + if (origin.length != 2 || shape.length != 2) { + throw new IllegalArgumentException("the length of origin and shape must be 2"); + } + this.xRange = new Range(origin[0], origin[0] + shape[0]); + this.yRange = new Range(origin[1], origin[1] + shape[1]); + } + + public int[] getOrigin() { + return new int[] {xRange.getStart(), yRange.getStart()}; + } + + public int[] getShape() { + return new int[] {xRange.getSize(), yRange.getSize()}; + } + + public Range getxRange() { + return xRange; + } + + public void setxRange(Range xRange) { + this.xRange = xRange; + } + + public Range getyRange() { + return yRange; + } + + public void setyRange(Range yRange) { + this.yRange = yRange; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof Plane) { + return xRange.equals(((Plane) o).getxRange()) && yRange.equals(((Plane) o).getyRange()); + } + return false; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Point.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Point.java new file mode 100644 index 0000000..bf7734f --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Point.java @@ -0,0 +1,39 @@ +package com.aliyun.tablestore.grid.model.grid; + +public class Point { + + private int x; + private int y; + + public Point(int x, int y) { + this.x = x; + this.y = y; + } + + public int getX() { + return x; + } + + public void setX(int x) { + this.x = x; + } + + public int getY() { + return y; + } + + public void setY(int y) { + this.y = y; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof Point) { + return x == ((Point) o).x && y == ((Point) o).y; + } + return false; + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Range.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Range.java new file mode 100644 index 0000000..d017fb3 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/model/grid/Range.java @@ -0,0 +1,53 @@ +package com.aliyun.tablestore.grid.model.grid; + +public class Range { + + private int start; + private int end; + + public Range(int end) { + this.start = 0; + this.end = end; + } + + public Range(int start, int end) { + this.start = start; + this.end = end; + } + + public int getStart() { + return start; + } + + public void setStart(int start) { + this.start = start; + } + + public int getEnd() { + return end; + } + + public void setEnd(int end) { + this.end = end; + } + + public int getSize() { + return this.end - this.start; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof Range) { + return start == ((Range) o).start && end == ((Range) o).end; + } + return false; + } + + @Override + public String toString() { + return ("[" + start + ", " + end + ")"); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/BlockUtil.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/BlockUtil.java new file mode 100644 index 0000000..f8a9bf5 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/BlockUtil.java @@ -0,0 +1,90 @@ +package com.aliyun.tablestore.grid.utils; + +import com.aliyun.tablestore.grid.model.grid.Grid2D; +import com.aliyun.tablestore.grid.model.grid.Plane; +import com.aliyun.tablestore.grid.model.grid.Point; +import com.aliyun.tablestore.grid.model.grid.Range; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class BlockUtil { + + public static List splitGrid2DToBlocks(Grid2D grid2D, int xSplitCount, int ySplitCount) throws InvalidRangeException { + Array array = grid2D.toArray(); + int blockXSize = (grid2D.getPlane().getxRange().getSize() - 1) / xSplitCount + 1; + int blockYSize = (grid2D.getPlane().getyRange().getSize() - 1) / ySplitCount + 1; + List result = new ArrayList(); + for (int i = 0; i < xSplitCount; i++) { + int startX = i * blockXSize; + int endX = Math.min(grid2D.getPlane().getxRange().getSize(), startX + blockXSize); + if (startX >= grid2D.getPlane().getxRange().getSize()) { + break; + } + for (int j = 0; j < ySplitCount; j++) { + int startY = j * blockYSize; + int endY = Math.min(grid2D.getPlane().getyRange().getSize(), startY + blockYSize); + if (startY >= grid2D.getPlane().getyRange().getSize()) { + break; + } + int[] origin = new int[] { startX, startY }; + int[] shape = new int[] { endX - startX, endY - startY }; + Array section = array.section(origin, shape); + Grid2D block = new Grid2D(section.getDataAsByteBuffer(), grid2D.getDataType(), origin, shape); + result.add(block); + } + } + return result; + } + + public static List calcBlockPointsCanCoverSubPlane(Plane plane, Plane subPlane, int xSplitCount, int ySplitCount) { + int blockXSize = (plane.getxRange().getSize() - 1) / xSplitCount + 1; + int blockYSize = (plane.getyRange().getSize() - 1) / ySplitCount + 1; + + Range xBlockIndexRange = new Range( + subPlane.getxRange().getStart() / blockXSize, + (subPlane.getxRange().getEnd() - 1) / blockXSize + 1); + Range yBlockIndexRange = new Range( + subPlane.getyRange().getStart() / blockYSize, + (subPlane.getyRange().getEnd() - 1) / blockYSize + 1); + + List points = new ArrayList(); + for (int xIdx = xBlockIndexRange.getStart(); xIdx < xBlockIndexRange.getEnd(); xIdx++) { + for (int yIdx = yBlockIndexRange.getStart(); yIdx < yBlockIndexRange.getEnd(); yIdx++) { + Point point = new Point(xIdx * blockXSize, yIdx * blockYSize); + points.add(point); + } + } + return points; + } + + public static Grid2D buildGrid2DFromBlocks(Plane plane, DataType dataType, List blocks, byte[] buffer, int pos) { + int size = plane.getxRange().getSize() * plane.getyRange().getSize() * dataType.getSize(); + if (buffer.length - pos < size) { + throw new IllegalArgumentException("buffer not enough"); + } + int count = 0; + for (Grid2D block : blocks) { + Plane blockPlane = block.getPlane(); + for (int x = Math.max(blockPlane.getxRange().getStart(), plane.getxRange().getStart()); + x < Math.min(blockPlane.getxRange().getEnd(), plane.getxRange().getEnd()); x++) { + for (int y = Math.max(blockPlane.getyRange().getStart(), plane.getyRange().getStart()); + y < Math.min(blockPlane.getyRange().getEnd(), plane.getyRange().getEnd()); y++) { + int posInBlock = dataType.getSize() * ((x - blockPlane.getxRange().getStart()) * (blockPlane.getyRange().getSize()) + (y - blockPlane.getyRange().getStart())) ; + int posInData = dataType.getSize() * ((x - plane.getxRange().getStart()) * plane.getyRange().getSize() + (y - plane.getyRange().getStart())); + System.arraycopy(block.getDataAsByteArray(), posInBlock, buffer, pos + posInData, dataType.getSize()); + count += dataType.getSize(); + } + } + } + if (count != size) { + throw new RuntimeException("the blocks does not contain enough data"); + } + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, pos, size); + return new Grid2D(byteBuffer, dataType, plane.getOrigin(), plane.getShape()); + } +} diff --git a/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/ValueUtil.java b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/ValueUtil.java new file mode 100644 index 0000000..62fc857 --- /dev/null +++ b/tablestore-grid-master/src/main/java/com/aliyun/tablestore/grid/utils/ValueUtil.java @@ -0,0 +1,47 @@ +package com.aliyun.tablestore.grid.utils; + +import com.alicloud.openservices.tablestore.model.ColumnValue; + +public class ValueUtil { + + public static ColumnValue toColumnValue(Object value) { + if (value instanceof Long) { + return ColumnValue.fromLong((Long) value); + } else if (value instanceof Integer) { + return ColumnValue.fromLong(((Integer) value).longValue()); + } else if (value instanceof Double) { + return ColumnValue.fromDouble((Double) value); + } else if (value instanceof String) { + return ColumnValue.fromString((String) value); + } else if (value instanceof Boolean) { + return ColumnValue.fromBoolean((Boolean) value); + } else if (value instanceof byte[]) { + return ColumnValue.fromBinary((byte[]) value); + } else { + throw new IllegalArgumentException("unsupported type: " + value.getClass()); + } + } + + public static Object toObject(ColumnValue value) { + switch (value.getType()) { + case INTEGER: { + return value.asLong(); + } + case STRING: { + return value.asString(); + } + case BOOLEAN: { + return value.asBoolean(); + } + case DOUBLE: { + return value.asDouble(); + } + case BINARY: { + return value.asBinary(); + } + default: { + throw new RuntimeException("unexpected"); + } + } + } +} diff --git a/tablestore-grid-master/src/main/resources/META-INF/MANIFEST.MF b/tablestore-grid-master/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..0953a6b --- /dev/null +++ b/tablestore-grid-master/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Class-Path: httpservices-5.5.3.jar re2j-1.3.jar udunits-5.5.3.jar jsr305 + -3.0.2.jar tablestore-5.16.0-jar-with-dependencies.jar cdm-core-5.5.3.j + ar j2objc-annotations-1.3.jar disruptor-3.4.4.jar logback-core-1.2.12.j + ar flatbuffers-java-1.11.0.jar jcommander-1.78.jar httpcore-4.4.13.jar + slf4j-api-1.7.28.jar listenablefuture-9999.0-empty-to-avoid-conflict-wi + th-guava.jar joda-time-2.10.3.jar lombok-1.18.30.jar commons-io-2.11.0. + jar commons-codec-1.11.jar protobuf-java-3.19.3.jar jdom2-2.0.6.jar fai + lureaccess-1.0.1.jar checker-qual-3.5.0.jar jj2000-5.4.jar httpclient-4 + .5.13.jar commons-logging-1.2.jar guava-30.1-jre.jar gson-2.8.5.jar err + or_prone_annotations-2.3.4.jar logback-classic-1.2.12.jar httpmime-4.5. + 13.jar grib-5.5.3.jar + diff --git a/tablestore-grid-master/src/main/resources/logback.xml b/tablestore-grid-master/src/main/resources/logback.xml new file mode 100644 index 0000000..9de982f --- /dev/null +++ b/tablestore-grid-master/src/main/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + %highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n + + + + debug + + + + + + + + + + + diff --git a/tablestore-grid-master/src/test/java/TestReadGrib2ByNC.java b/tablestore-grid-master/src/test/java/TestReadGrib2ByNC.java new file mode 100644 index 0000000..259bfea --- /dev/null +++ b/tablestore-grid-master/src/test/java/TestReadGrib2ByNC.java @@ -0,0 +1,60 @@ +import ucar.ma2.Array; +import ucar.ma2.Index; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * Unit test for simple App. + */ +public class TestReadGrib2ByNC { + public static void main(String[] args){ + // String ncFile = "surface.nc"; + String file = "C:\\Users\\shi_y\\Desktop\\java_learn\\data_download\\GFSData\\UTC-20230910\\BJT-20230911-0200.grib2"; + try (NetcdfFile ncFile = NetcdfFiles.open(file)) { + // 打印文件相关信息 + // System.out.println(ncfile.toString()); + + // 获取时间变量 + Variable timeVar = ncFile.findVariable("time"); + String units = timeVar.findAttribute("units").getStringValue(); + System.out.println(units); + // 读取时间变量 surface.nc的单位是"hours since 1900-01-01 00:00:00.0" + System.out.println(timeVar.getFileTypeId()); + double[] timeArray = (double[]) timeVar.read().copyTo1DJavaArray(); + + DateTimeFormatter unitsFormat = DateTimeFormatter.ofPattern("'Hour since 'yyyy-MM-dd'T'HH:mm:ss'Z'"); + LocalDateTime unitsDateTime = LocalDateTime.parse(units, unitsFormat).plusHours((long) timeArray[0]); + System.out.println(unitsDateTime); + // 指定变量名读取变量 + Variable t2m = ncFile.findVariable("Temperature_isobaric"); + + // 打印变量的相关信息 + System.out.println(t2m.toString()); + + if (t2m != null) { + System.out.println(t2m.findDimensionIndex("lon")); + } + // 读取所有的数据 + assert t2m != null; + Array v = t2m.read("0, 0, 10:20, 20:30").reduce(0).reduce(0); + Index index = v.getIndex(); // 用于跟踪当前索引的位置 + index = index.set(1); + System.out.println(v.getShort(index)); + + + } catch (IOException ioe) { + // Handle less-cool exceptions here + throw new RuntimeException(ioe); + } catch (InvalidRangeException e) { + throw new RuntimeException(e); + } + } + +} + diff --git a/weather-service/pom.xml b/weather-service/pom.xml new file mode 100644 index 0000000..4bf56f9 --- /dev/null +++ b/weather-service/pom.xml @@ -0,0 +1,159 @@ + + + 4.0.0 + com.htfp.weather + weather-service + 0.0.1-SNAPSHOT + weather-service + weather-service + + 1.8 + UTF-8 + UTF-8 + 2.6.13 + + + + + unidata-all + Unidata All + https://artifacts.unidata.ucar.edu/repository/unidata-all/ + + + + + + edu.ucar + cdm-core + 5.5.3 + + + edu.ucar + grib + 5.5.3 + + + com.aliyun.openservices + tablestore + 5.16.0 + jar-with-dependencies + + + org.apache.httpcomponents + httpasyncclient + + + + + + ai.djl + api + 0.27.0 + + + ai.djl.pytorch + pytorch-engine + 0.27.0 + + + commons-io + commons-io + 2.11.0 + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + + com.squareup.okhttp3 + okhttp + 4.9.0 + + + + com.google.android + android + + + + + com.aliyun.tablestore + tablestore-grid + 1.0-SNAPSHOT + + + + org.projectlombok + lombok + true + + + + javax.validation + validation-api + 2.0.1.Final + + + org.hibernate + hibernate-validator + 6.0.16.Final + + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + com.htfp.weather.WeatherServiceApplication + true + + + + repackage + + repackage + + + + + + + + diff --git a/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java new file mode 100644 index 0000000..42516f8 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java @@ -0,0 +1,19 @@ +package com.htfp.weather; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@SpringBootApplication +public class WeatherServiceApplication { + + public static void main(String[] args) { + try { + SpringApplication.run(WeatherServiceApplication.class, args); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/config/Config.java b/weather-service/src/main/java/com/htfp/weather/config/Config.java new file mode 100644 index 0000000..c539435 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/config/Config.java @@ -0,0 +1,20 @@ +package com.htfp.weather.config; + +import com.htfp.weather.griddata.common.TableConfig; +import com.htfp.weather.griddata.common.TableConfigBean; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +/** + * @Author : shiyi + * @Date : 2024/1/22 19:02 + * @Description : + */ +@Component("config") +public class Config { + @Resource + TableConfigBean tableConfigBean; + +} diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java new file mode 100644 index 0000000..3537993 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java @@ -0,0 +1,51 @@ +package com.htfp.weather.download; + +import lombok.Data; +import lombok.ToString; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.List; + +/** + * @author shiyi + */ +@Data +@ToString +public abstract class BaseDataDownloader { + /**起始时间，北京时UTC+8，格式示例：20230901T08:00*/ + private String startTimeStr; + private LocalDateTime startTime; + private String refTimeStr; + /** 数据的实际起报UTC时间 */ + private OffsetDateTime refTime; + + /** + * 获取所有下载文件的相关信息 + */ + public abstract List getFilesInfo(); + + /** + * 单个文件下载 + * @param fileInfo: 目标文件的下载信息 + * @return 下载成功/失败 + * @throws IOException + */ + public abstract boolean download(FileInfo fileInfo) throws IOException; + + /** + * 下载所有目标文件 + * + * @param filesInfo + * @return + */ + public abstract boolean downloadAll(List fileInfoList); + + + public BaseDataDownloader() { + + } + +} + diff --git a/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java b/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java new file mode 100644 index 0000000..c8c8d23 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/download/FileInfo.java @@ -0,0 +1,31 @@ +package com.htfp.weather.download; + +import lombok.Data; +import lombok.ToString; + +/** + * @author shiyi + */ + +@Data +@ToString +public class FileInfo { + /**保存文件名*/ + private String fileName; + /**数据起报时间*/ + private String refTimeStr; + /**预报时效*/ + private Integer forecastHour; + /**预报时效对应的北京时间*/ + private String forecastBJTimeStr; + /**预报时效对应的UTC时间*/ + private String forecastUTCTimeStr; + /**下载链接*/ + private String url; + + private String savePath; + + public FileInfo() { + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java new file mode 100644 index 0000000..43cd91b --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDataConfig.java @@ -0,0 +1,154 @@ +package com.htfp.weather.download; + +import com.htfp.weather.info.GfsDownloadVariableEnum; +import com.htfp.weather.info.GfsLevelsEnum; +import com.htfp.weather.utils.JSONUtils; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Objects; + +/** + * @Author : shiyi + * @Date : 2024/2/28 13:53 + * @Description : GFS数据下载配置 + */ +@Slf4j @Data +@Component +public class GfsDataConfig { + @NotNull(message = "下载范围 (duration) 不能为空") + @Min(value = 1, message = "下载范围 (duration) 需大于等于1小时") + @Max(value = 36, message = "下载范围 (duration) 不能超过36小时") + private int duration = 5; + + @NotNull(message = "最小经度不能为空") + @Min(value = -180, message = "经度最小值为-180（西经180°）") + @Max(value = 180, message = "经度最大值为180（东经180°）") + private Double minLon; + + @NotNull(message = "最大经度不能为空") + @Min(value = -180, message = "经度最小值为-180（西经180°）") + @Max(value = 180, message = "经度最大值为180（东经180°）") + private Double maxLon; + + @NotNull(message = "最小纬度不能为空") + @Min(value = -90, message = "纬度最小值为-90（南纬90°）") + @Max(value = 90, message = "纬度最大值为90（北纬90°）") + private Double minLat; + + @NotNull(message = "最大纬度不能为空") + @Min(value = -90, message = "纬度最小值为-90（南纬90°）") + @Max(value = 90, message = "纬度最大值为90（北纬90°）") + private Double maxLat; + + @NotNull(message = "分辨率 (resolution) 不能为空") + private Double resolution = 0.25; + + @Valid + @NotEmpty(message = "变量列表 (variables) 不能为空") + // @EnumValid(enumClass = GfsDataVariableEnum.class, message = "变量列表 (variables) 必须为 GfsDataVariableEnum 枚举值") + private ArrayList variables; + + @Valid + @NotEmpty(message = "气压高度列表 (pressureLevels) 不能为空") + // @EnumValid(enumClass = GfsLevelsEnum.class, message = "变量列表 (pressureLevels) 必须为 GfsLevelsEnum 枚举值") + private int[] pressureLevels; + private int[] heightLevels = new int[]{2, 10, 20, 30, 40, 50, 80, 100}; + @NotEmpty(message = "数据存储路径 (saveRoot) 不能为空") + private String saveRoot; + + private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config")).getPath() + "/gfsDataConfig.json"; + public void valid() { + // TODO 2024/4/21: 校验 + if (minLon >= maxLon) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度"); + } + if (minLat >= maxLat) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度"); + } + ArrayList wrongVariables = new ArrayList<>(); + ArrayList wrongLevels = new ArrayList<>(); + for (String variable : variables) { + if (!GfsDownloadVariableEnum.contains(variable)) { + wrongVariables.add(variable); + } + } + if (!wrongVariables.isEmpty()) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, " variables = " +wrongVariables + "非 GfsLevelsEnum 枚举值"); + } + + for (Integer level : pressureLevels) { + if (!GfsLevelsEnum.contains(level)) { + wrongLevels.add(level); + } + } + if (!wrongLevels.isEmpty()) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, " pressureLevels = " +wrongLevels + "非 GfsLevelsEnum 枚举值"); + } + } + @PostConstruct + public void init() { + readConfig(); + } + public void readConfig() { + try (InputStream f = new FileInputStream(configPath)){ + GfsDataConfig gfsDataConfig = JSONUtils.json2pojo(IOUtils.toString(f), GfsDataConfig.class); + this.duration = gfsDataConfig.getDuration(); + this.minLon = gfsDataConfig.getMinLon(); + this.maxLon = gfsDataConfig.getMaxLon(); + this.minLat = gfsDataConfig.getMinLat(); + this.maxLat = gfsDataConfig.getMaxLat(); + this.resolution = gfsDataConfig.getResolution(); + this.variables = gfsDataConfig.getVariables(); + this.pressureLevels = gfsDataConfig.getPressureLevels(); + this.saveRoot = gfsDataConfig.getSaveRoot(); + log.info("[config] 下载配置初始化: {}", configPath); + } catch (NoSuchFileException e) { + log.error("[config] 配置文件{}不存在", configPath, e); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + + public void writeConfig(GfsDataConfig updateConfig) throws IOException { + writeConfig(updateConfig, configPath); + } + public void writeConfig(GfsDataConfig updateConfig, String filePath) throws IOException { + this.duration = updateConfig.getDuration(); + this.minLon = updateConfig.getMinLon(); + this.maxLon = updateConfig.getMaxLon(); + this.minLat = updateConfig.getMinLat(); + this.maxLat = updateConfig.getMaxLat(); + this.resolution = updateConfig.getResolution(); + this.variables = updateConfig.getVariables(); + this.pressureLevels = updateConfig.getPressureLevels(); + this.saveRoot = updateConfig.getSaveRoot(); + // 如果目录不存在则创建 + File destDir = new File(saveRoot); + if (!destDir.exists()) { + boolean mkdirs = destDir.mkdirs(); + if (!mkdirs) { + throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "目录创建失败: " + saveRoot); + } + } + JSONUtils.pojo2jsonFile(this, filePath); + log.info("配置文件 {} 更新为: {}", filePath, updateConfig); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java new file mode 100644 index 0000000..6ca9fe5 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java @@ -0,0 +1,265 @@ +package com.htfp.weather.download; + +import com.htfp.weather.info.Constant; +import com.htfp.weather.utils.DateTimeUtils; +import com.htfp.weather.utils.HttpClientUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; + +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + + +/** + * @author shiyi + */ +@Slf4j +@Component("gfsDownloader") +@DependsOn("gfsDataConfig") +public class GfsDownloader extends BaseDataDownloader { + @Resource + GfsDataConfig gfsDataConfig; + private final ExecutorService executorService = Executors.newFixedThreadPool(5); + private int hourDiff; + private OffsetDateTime realUpdateDateTime; + + public GfsDownloader(GfsDataConfig gfsDataConfig) { + super(); + this.gfsDataConfig = gfsDataConfig; + } + + /** + * 初始化下载起始时间 + */ + public void iniTimeSetting() { + // 当前时间 + this.setStartTime(LocalDateTime.now()); + this.setStartTimeStr(LocalDateTime.now().format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING))); + // 以下时间均为UTC时间 + OffsetDateTime utcStartTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now()); + realUpdateDateTime = getAvailableUpdateDateTime(utcStartTime); + this.setRefTime(realUpdateDateTime); + this.setRefTimeStr(realUpdateDateTime.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))); + // 起报点和实际请求时间的差异 + + this.hourDiff = (int) Duration.between(realUpdateDateTime, utcStartTime).toHours(); + log.info("[GFS Download] 起始为当前本地时间：{}", this.getStartTime()); + log.info("[GFS Download] GFS最近更新时间UTC：{}，本地时间: {}", realUpdateDateTime, DateTimeUtils.getLocalZoneDateTime(realUpdateDateTime)); + } + + @Override + public boolean download(FileInfo fileInfo) { + try { + String url = fileInfo.getUrl(); + File destDir = new File(fileInfo.getSavePath()); + if (!destDir.exists()) { + destDir.mkdirs(); + } + File fileOut = new File(destDir, fileInfo.getFileName()); + // if (fileOut.exists()) { + // log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath()); + // return true; + // } + log.info("[GFS Download] 文件下载中，保存至 {}", fileOut); + // FileOutputStream fos = new FileOutputStream(filePath); + // try { + // ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream()); + // fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + // } catch (ConnectException e) { + // log.error("连接超时:{}", url); + // } + // + // fos.flush(); + // fos.close(); + FileUtils.copyURLToFile(new URL(url), fileOut); + log.info("[GFS Download] 文件下载成功: {}", fileOut); + return fileValid(fileOut.getAbsolutePath()); + } catch (Exception e) { + log.error("[GFS Download] 文件 {} 下载失败：{}", fileInfo, e.getMessage()); + return false; + } + } + private boolean fileValid(String file) { + try (NetcdfFile ncFile = NetcdfFiles.open(file)) { + ncFile.getLocation(); + return true; + } catch (IOException e){ + return false; + } + } + @Override + public boolean downloadAll(List fileInfoList) { + log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size()); + List> futures = new ArrayList<>(); + for (FileInfo fileInfo : fileInfoList) { + futures.add(executorService.submit(() -> download(fileInfo))); + } + boolean allCompleted = true; + for (Future future : futures) { + try { + allCompleted = allCompleted && future.get(); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + return allCompleted; + } + + @Override + public List getFilesInfo() { + // 时间选项填充 + String lonlatBoxStr = null; + String levelsStr = null; + String variablesStr = null; + try { + lonlatBoxStr = fillUrlForLonLat(); + levelsStr = fillUrlForLevels(); + variablesStr = fillUrlForVariables(); + } catch (Exception e) { + log.error("[GFS Download] 文件下载参数生成错误"); + throw new RuntimeException(e); + } + + // 预报数据时间分辨率 + final int forecastStep = 1; + // 分辨率为3h，整数截断；包括目标时刻本身 + final int nFiles = 1 + gfsDataConfig.getDuration() / forecastStep; + // 存储文件信息 + List fileInfoList = new ArrayList<>(nFiles); + for (int i = 0; i < nFiles; i++) { + FileInfo fileInfo = new FileInfo(); + int forecastHour = i * forecastStep + hourDiff; + fileInfo.setForecastHour(forecastHour); + String baseURL = getBaseURL(forecastHour, levelsStr, variablesStr, lonlatBoxStr); + fileInfo.setUrl(baseURL); + fileInfo.setRefTimeStr(this.getRefTime().format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))); + fileInfo.setForecastUTCTimeStr( + this.getRefTime().plusHours(fileInfo.getForecastHour()) + .format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)) + ); + fileInfo.setForecastBJTimeStr( + DateTimeUtils.getLocalZoneDateTime(this.getRefTime().plusHours(fileInfo.getForecastHour())) + .format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING)) + ); + fileInfo.setFileName(String.format("%s-from-%s+%d.grib2", fileInfo.getForecastBJTimeStr(), fileInfo.getRefTimeStr(), forecastHour)); + fileInfo.setSavePath(gfsDataConfig.getSaveRoot() + "/" + fileInfo.getRefTimeStr()); + fileInfoList.add(fileInfo); + } + return fileInfoList; + } + + + private String getBaseURL(int forcastHour, String levelsStr, String variablesStr, String lonlatBoxStr) { + String baseFormat = "https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25_1hr.pl?dir=/gfs.%s/%02d/atmos&file=gfs.t%02dz.pgrb2.0p25.f%03d"; + + String realUpdateDateString = realUpdateDateTime.format(DateTimeFormatter.ofPattern("yyyyMMdd")); + int refHour = realUpdateDateTime.getHour(); + + String baseurl = String.format(baseFormat, realUpdateDateString, refHour, refHour, forcastHour); + StringJoiner stringJoiner = new StringJoiner("&"); + + return stringJoiner.add(baseurl).add(levelsStr).add(variablesStr).add(lonlatBoxStr).toString(); + } + + /** + * 获取距离指定时刻最近的更新时刻 + */ + private OffsetDateTime getAvailableUpdateDateTime(OffsetDateTime utcStartTime) { + + + // 获取临近的更新点 + int hour = utcStartTime.getHour(); + int[] updateHours = {0, 6, 12, 18}; + int nearUpdateHour = 0; + for (int i : updateHours) { + if (i < hour) { + nearUpdateHour = i; + } + } + OffsetDateTime nearUpdateDateTime = utcStartTime.withHour(nearUpdateHour).withMinute(0).withSecond(0).withNano(0); + + OffsetDateTime now = OffsetDateTime.now(); + // 如果最近的更新点还没有更新数据，那么返回前一个更新点 + // if (isGfsUpdated(nearUpdateDateTime)) { + if (now.isAfter(nearUpdateDateTime.plusHours(4))) { + // GFS延迟4小时才能获取到数据，比如04时更新00时的数据 + return nearUpdateDateTime; + } else { + return nearUpdateDateTime.minusHours(6); + } + + } + + /** + * 判断GFS是否更新 + * + * @return + */ + public boolean isGfsUpdated(OffsetDateTime utcUpdateTime) { + String timeString = utcUpdateTime.format(DateTimeFormatter.ofPattern("'gfs.'yyyyMMdd/HH")); + String url = String.format("https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/%s/", timeString); + try { + String httpString = HttpClientUtils.sendGet(url, null); + if (StringUtils.isEmpty(httpString)) { + return false; + } + // FIXME 2024/5/17: 可能遇到正在更新，文件夹存在，但是文件不一定存在 + return httpString.contains("atmos/") && httpString.contains("wave/"); + } catch (Exception e) { + return false; + } + } + + private String fillUrlForLonLat() { + // 根据传入参数生成对应的URL请求信息，生成字符串 + Double topLat = gfsDataConfig.getMaxLat(); + Double leftLon = gfsDataConfig.getMinLon(); + Double rightLon = gfsDataConfig.getMaxLon(); + Double bottomLat = gfsDataConfig.getMinLat(); + return String.format("subregion=&toplat=%.2f&leftlon=%.2f&rightlon=%.2f&bottomlat=%.2f", topLat, leftLon, rightLon, bottomLat); + } + + private String fillUrlForLevels() { + int[] pressureLevels = gfsDataConfig.getPressureLevels(); + if (pressureLevels.length == 0) { + throw (new IllegalArgumentException("至少指定一个层次")); + } + StringJoiner stringJoiner = new StringJoiner("&"); + for (int pres : pressureLevels) { + stringJoiner.add(String.format("lev_%d_mb=on", pres)); + } + // 添加地面2m和10m的信息 + int[] heightLevels = gfsDataConfig.getHeightLevels(); + for (int height : heightLevels) { + stringJoiner.add(String.format("lev_%d_m_above_ground", height)); + } + return stringJoiner.toString(); + } + + private String fillUrlForVariables() { + ArrayList variables = gfsDataConfig.getVariables(); + StringJoiner stringJoiner = new StringJoiner("&"); + if (variables.isEmpty()) { + throw (new IllegalArgumentException("至少指定一个下载变量")); + } + for (String variable : variables) { + stringJoiner.add(String.format("var_%s=on", variable)); + } + return stringJoiner.toString(); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java new file mode 100644 index 0000000..b76e45e --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/GfsVariableNameEnum.java @@ -0,0 +1,54 @@ +package com.htfp.weather.griddata.common; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @Author : shiyi + * @Date : 2024/1/15 14:54 + * @Description : 文件变量名和表中变量名的映射 + */ +public enum GfsVariableNameEnum { + // + TEMP("temp", "Temperature_isobaric"), + CLOUD("cloud", "Total_cloud_cover_isobaric"), + WIND_SPEED("windSpeed", "Wind_speed_isobaric"), + WIND360("wind360", "Wind_direction_isobaric"), + ; + private static final ConcurrentHashMap variableName = new ConcurrentHashMap() {{ + put("temp", "Temperature_isobaric"); + put("cloud", "Total_cloud_cover_isobaric"); + put("windSpeed", "Wind_speed_isobaric"); + put("wind360", "Wind_direction_isobaric"); + }}; + String nameInApi; + String nameInFile; + + GfsVariableNameEnum(String nameInApi, String nameInFile) { + this.nameInApi = nameInApi; + this.nameInFile = nameInFile; + } + + public String getNameInApi() { + return nameInApi; + } + + public String getNameInFile() { + return nameInFile; + } + public static String getGfsVariableName(GfsVariableNameEnum gfsVariableNameEnum) { + return variableName.get(gfsVariableNameEnum.nameInApi); + } + public static String getGfsVariableName(String tableVariableName) { + return variableName.get(tableVariableName); + } + + public static List getVariableNamesInApi() { + return new ArrayList<>(variableName.keySet()); + } + + public static List getVariableNamesInFile() { + return new ArrayList<>(variableName.values()); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java new file mode 100644 index 0000000..c7633a2 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfig.java @@ -0,0 +1,108 @@ +package com.htfp.weather.griddata.common; + +import com.google.gson.Gson; +import com.htfp.weather.download.GfsDataConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import ucar.ma2.DataType; + +import javax.annotation.PostConstruct; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @Author : shiyi + * @Date : 2024/1/15 19:10 + * @Description : 表格meta配置 + */ +@Slf4j @Component +public class TableConfig { + @Autowired + GfsDataConfig dataConfig; + /** + * 表名和index名 + */ + public static String DATA_TABLE_NAME; + public static String META_TABLE_NAME; + public static String DATA_INDEX_NAME; + public static String META_INDEX_NAME; + + /** + * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 + */ + public static String DATA_DIR; + public static List VARIABLE_LIST; + public static int lonSize; + public static List lonList; + public static int latSize; + public static List latList; + public static int levSize; + public static List levList; + public static int timeSizeMax = 72; + public static List timeList; + public static DataType DATA_TYPE = DataType.FLOAT; + + // 数据过期时间，单位为秒 + public static int TIME_TO_LIVE; + @PostConstruct + private void initTableConfig() { + initDatasetConfig(); + initLonList(); + initLatList(); + initLevList(); + } + + + /** + * 初始化表格配置 + */ + public static void initDatasetConfig() { + log.info("init table config..."); + String path = Objects.requireNonNull(TableConfig.class.getClassLoader().getResource("config/tableConf.json")).getPath(); + try (InputStream f = new FileInputStream(path)){ + Gson gson = new Gson(); + Map map = gson.fromJson(IOUtils.toString(f), Map.class); + DATA_TABLE_NAME = (String) map.get("dataTableName"); + META_TABLE_NAME = (String) map.get("metaTableName"); + DATA_INDEX_NAME = (String) map.get("dataIndexName"); + META_INDEX_NAME = (String) map.get("metaIndexName"); + DATA_DIR = (String) map.get("dataDir"); + VARIABLE_LIST = (ArrayList) map.get("variableList"); + lonSize = ((Double) map.get("lonSize")).intValue(); + latSize = ((Double) map.get("latSize")).intValue(); + levSize = ((Double) map.get("levSize")).intValue(); + TIME_TO_LIVE = (int) (((Double) map.get("timeToLive")) * 24 *3600 ) ; // 配置文件的单位为天 + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private void initLonList() { + double lonStart = dataConfig.getMinLon(); + double res = dataConfig.getResolution(); + lonList = new ArrayList<>(); + for (int i = 0; i < lonSize; i++) { + lonList.add(lonStart + i * res); + } + } + + private void initLatList() { + double latStart = dataConfig.getMinLat(); + double res = dataConfig.getResolution(); + latList = new ArrayList<>(); + for (int i = 0; i < latSize; i++) { + latList.add(latStart + i * res); + } + } + + private void initLevList() { + levList = Arrays.stream(dataConfig.getPressureLevels()).boxed().collect(Collectors.toList()); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java new file mode 100644 index 0000000..523324a --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableConfigBean.java @@ -0,0 +1,144 @@ +package com.htfp.weather.griddata.common; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; +import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.utils.JSONUtils; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import ucar.ma2.DataType; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@Data @Component @Slf4j +@JsonIgnoreProperties(ignoreUnknown = true) +public class TableConfigBean { + @Resource + @JsonIgnore + GfsDataConfig dataConfig; + @Value("duration") + public String dataTableName; + public String metaTableName; + public String dataIndexName; + public String metaIndexName; + + /** + * 数据属性：ID、本地数据文件名、目标变量名、变量维度大小、变量类型 + */ + public String dataDir; + public List variableList; + public int lonSize; + public int latSize; + public int levSize; + public int timeToLive; + + @JsonIgnore + public int timeSizeMax = 99; + @JsonIgnore + public double[] lonList; + @JsonIgnore + public double[] latList; + @JsonIgnore + public int[] levList; + // @JsonIgnore + // public List forecastTimeList; // 预报时效列表 + @JsonIgnore + public DataType dataType = DataType.FLOAT; + + + // note 修改作用范围在classes目录下，resource文件下不改变 + @JsonIgnore + private final String configPath = Objects.requireNonNull(this.getClass().getClassLoader().getResource("config/tableConf.json")).getPath(); + @PostConstruct + private void initTableConfig() { + readConfig(); + initLonList(); + initLatList(); + initLevList(); + } + + public void readConfig() { + // String pathSeparator = System.getProperty("file.separator"); + try (InputStream f = new FileInputStream(configPath)){ + String jsonStr = IOUtils.toString(f, StandardCharsets.UTF_8); + TableConfigBean tableConfig = JSONUtils.json2pojo(jsonStr, TableConfigBean.class); + this.dataTableName = tableConfig.getDataTableName(); + this.metaTableName = tableConfig.getMetaTableName(); + this.dataIndexName = tableConfig.getDataIndexName(); + this.metaIndexName = tableConfig.getMetaIndexName(); + this.variableList = tableConfig.getVariableList(); + this.dataDir = tableConfig.getDataDir(); + this.lonSize = tableConfig.getLonSize(); + this.latSize = tableConfig.getLatSize(); + this.levSize = dataConfig.getPressureLevels().length; + this.timeToLive = tableConfig.getTimeToLive(); + log.info("[config] 读取 DataTable 配置 : {}", configPath); + } catch (NoSuchFileException e) { + log.error("[config] 配置文件{}不存在", configPath, e); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public void writeConfig(TableConfigBean updateConfig) throws IOException { + writeConfig(updateConfig, configPath); + } + public void writeConfig(TableConfigBean updateConfig, String filePath) throws IOException { + this.dataTableName = updateConfig.getDataTableName(); + this.metaTableName = updateConfig.getMetaTableName(); + this.dataIndexName = updateConfig.getDataIndexName(); + this.metaIndexName = updateConfig.getMetaIndexName(); + this.variableList = updateConfig.getVariableList(); + this.dataDir = updateConfig.getDataDir(); + this.lonSize = updateConfig.getLonSize(); + this.latSize = updateConfig.getLatSize(); + this.levSize = updateConfig.getLevSize(); + // // FIXME 2024/5/10: 维度信息取决于数据下载，目前需要两个配置一起更新 + this.timeToLive = updateConfig.getTimeToLive(); + JSONUtils.pojo2jsonFile(this, filePath); + log.info("配置文件 {} 更新为: {}", filePath, updateConfig); + } + + public void valid() { + } + + + private void initLonList() { + double lonStart = dataConfig.getMinLon(); + double res = dataConfig.getResolution(); + lonList = new double[lonSize]; + for (int i = 0; i < lonSize; i++) { + lonList[i] = lonStart + i * res; + } + } + + private void initLatList() { + double latStart = dataConfig.getMinLat(); + double res = dataConfig.getResolution(); + latList = new double[latSize]; + for (int i = 0; i < latSize; i++) { + latList[i] = latStart + i * res; + } + } + + private void initLevList() { + levList = dataConfig.getPressureLevels(); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/common/TableStoreConf.java b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableStoreConf.java new file mode 100644 index 0000000..b221697 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/common/TableStoreConf.java @@ -0,0 +1,15 @@ +package com.htfp.weather.griddata.common; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration("tableStoreConf") +@ConfigurationProperties("tablestore") +public class TableStoreConf { + private String endpoint; + private String accessId; + private String accessKey; + private String instanceName; +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java new file mode 100644 index 0000000..619692c --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/BaseTableOperation.java @@ -0,0 +1,44 @@ +package com.htfp.weather.griddata.operation; + + +import com.aliyun.tablestore.grid.TableStoreGridConfig; +import com.aliyun.tablestore.grid.TableStoreGrid; +import com.google.common.collect.Tables; +import com.htfp.weather.griddata.common.TableConfig; +import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.common.TableStoreConf; +import org.checkerframework.checker.units.qual.A; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@DependsOn("tableStoreConf") +public abstract class BaseTableOperation{ + + protected TableStoreGrid tableStoreGrid; + // private String pathSeperator = "/"; + @Autowired + private TableConfigBean tableConfigBean; + @Autowired + private TableStoreConf tableStoreConf; + + @PostConstruct + public void init() { + TableStoreGridConfig config = new TableStoreGridConfig(); + config.setTableStoreEndpoint(tableStoreConf.getEndpoint()); + config.setAccessId(tableStoreConf.getAccessId()); + config.setAccessKey(tableStoreConf.getAccessKey()); + config.setTableStoreInstance(tableStoreConf.getInstanceName()); + config.setDataTableName(tableConfigBean.getDataTableName()); + config.setMetaTableName(tableConfigBean.getMetaTableName()); + tableStoreGrid = new TableStoreGrid(config); + } + + public void close() { + if (tableStoreGrid != null) { + tableStoreGrid.close(); + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java new file mode 100644 index 0000000..ca0cf7f --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/CreateTable.java @@ -0,0 +1,53 @@ +package com.htfp.weather.griddata.operation; + +import com.alicloud.openservices.tablestore.model.TableOptions; +import com.alicloud.openservices.tablestore.model.search.FieldSchema; +import com.alicloud.openservices.tablestore.model.search.FieldType; +import com.alicloud.openservices.tablestore.model.search.IndexSchema; +import com.htfp.weather.griddata.common.TableConfig; + + +import java.util.Arrays; + +/** + * @Author : shiyi + * @Date : 2024/1/15 15:10 + * @Description : 创建表 + */ +public class CreateTable extends BaseTableOperation { + + /** + * we must create store before we can use it. + * @throws Exception + */ + private void createStore() throws Exception { + TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE, 1); + this.tableStoreGrid.createStore(tableOptions); + } + + /** + * this example create an index which contains these columns: status, tag1, tag2, create_time. + * you can create an index which contains any other columns. + * + * @throws Exception + */ + private void createIndex() throws Exception { + IndexSchema indexSchema = new IndexSchema(); + indexSchema.setFieldSchemas(Arrays.asList( + new FieldSchema("status", FieldType.KEYWORD).setIndex(true).setEnableSortAndAgg(true), + new FieldSchema("create_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true), + new FieldSchema("ref_time", FieldType.LONG).setIndex(true).setEnableSortAndAgg(true) + )); + this.tableStoreGrid.createMetaIndex(TableConfig.META_INDEX_NAME, indexSchema); + } + + public static void main(String[] args) throws Exception { + CreateTable example = new CreateTable(); + try { + example.createStore(); + example.createIndex(); + } finally { + example.close(); + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java new file mode 100644 index 0000000..8f1224e --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/DataDeleter.java @@ -0,0 +1,130 @@ +package com.htfp.weather.griddata.operation; + +import com.alicloud.openservices.tablestore.AsyncClient; +import com.alicloud.openservices.tablestore.model.*; +import com.aliyun.tablestore.grid.GridDataDeleter; +import com.aliyun.tablestore.grid.core.RequestBuilder; +import com.aliyun.tablestore.grid.model.DeleteDataParam; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.htfp.weather.griddata.common.TableConfig; + +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.Field; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +/** + * @Author : shiyi + * @Date : 2024/2/2 16:53 + * @Description : 批量删除数据 + */ +@Slf4j +public class DataDeleter extends BaseTableOperation { + + /** 删除指定datasetId的所有数据 + * + * @param dataSetId + + * @return + * @throws Exception + */ + public void deleteDataset(String dataSetId) throws Exception { + GridDataDeleter deleter = tableStoreGrid.getDataDeleter(tableStoreGrid.getDataSetMeta(dataSetId)); + deleter.delete(dataSetId); + } + + private AsyncClient getClient() throws IllegalAccessException, NoSuchFieldException { + Field declaredField = this.tableStoreGrid.getClass().getDeclaredField("asyncClient"); + declaredField.setAccessible(true); + AsyncClient client = (AsyncClient) declaredField.get(this.tableStoreGrid); + return client; + } + private void batchWriteRow() throws Exception { + AsyncClient client = getClient(); + BatchWriteRowRequest batchWriteRowRequest = new BatchWriteRowRequest(); + + //构造rowDeleteChange1。 + // PrimaryKeyBuilder pk1Builder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + // pk1Builder.addPrimaryKeyColumn("_id", PrimaryKeyValue.fromString("pk")); + // //设置数据表名称。 + // RowDeleteChange rowDeleteChange1 = new RowDeleteChange("", pk1Builder.build()); + // //添加到batch操作中。 + // batchWriteRowRequest.addRowChange(rowDeleteChange1); + //构造rowDeleteChange2。 + String dataSetId = "UTC-20230910"; + GridDataSetMeta dataSetMeta = this.tableStoreGrid.getDataSetMeta(dataSetId); + Set refTimes = dataSetMeta.getForecastHours().stream().map(Integer::parseInt).collect(Collectors.toSet()); + for (int t=0;t<3;t++) { + for (int z=0; z<8; z++) { + DeleteDataParam param = new DeleteDataParam(TableConfig.DATA_TABLE_NAME, "UTC-20230910", "Total_cloud_cover_isobaric", t, z); + //添加到batch操作中。 + batchWriteRowRequest.addRowChange(RequestBuilder.buildDeleteDataRequest(param).getRowChange()); + } + } + + + + BatchWriteRowResponse response = client.batchWriteRow(batchWriteRowRequest, null).get(); + + System.out.println("是否全部成功：" + response.isAllSucceed()); + if (!response.isAllSucceed()) { + for (BatchWriteRowResponse.RowResult rowResult : response.getFailedRows()) { + System.out.println("失败的行：" + batchWriteRowRequest.getRowChange(rowResult.getTableName(), rowResult.getIndex()).getPrimaryKey()); + System.out.println("失败原因：" + rowResult.getError()); + } + /** + * 可以通过createRequestForRetry方法再构造一个请求对失败的行进行重试。此处只给出构造重试请求的部分。 + * 推荐的重试方法是使用SDK的自定义重试策略功能，支持对batch操作的部分行错误进行重试。设置重试策略后，调用接口处无需增加重试代码。 + */ + BatchWriteRowRequest retryRequest = batchWriteRowRequest.createRequestForRetry(response.getFailedRows()); + } + } + + private void getRange(String pk, String startPkValue, String endPkValue) throws NoSuchFieldException, IllegalAccessException, ExecutionException, InterruptedException { + AsyncClient client = getClient(); + + //设置数据表名称。 + RangeRowQueryCriteria rangeRowQueryCriteria = new RangeRowQueryCriteria(TableConfig.DATA_TABLE_NAME); + + //设置起始主键。 + PrimaryKeyBuilder primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + primaryKeyBuilder.addPrimaryKeyColumn(pk, PrimaryKeyValue.fromString(startPkValue)); + rangeRowQueryCriteria.setInclusiveStartPrimaryKey(primaryKeyBuilder.build()); + + //设置结束主键。 + primaryKeyBuilder = PrimaryKeyBuilder.createPrimaryKeyBuilder(); + primaryKeyBuilder.addPrimaryKeyColumn(pk, PrimaryKeyValue.fromString(endPkValue)); + rangeRowQueryCriteria.setExclusiveEndPrimaryKey(primaryKeyBuilder.build()); + + rangeRowQueryCriteria.setMaxVersions(1); + + System.out.println("GetRange的结果为："); + while (true) { + GetRangeResponse getRangeResponse = client.getRange(new GetRangeRequest(rangeRowQueryCriteria),null).get(); + for (Row row : getRangeResponse.getRows()) { + System.out.println(row); + } + + //如果NextStartPrimaryKey不为null，则继续读取。 + if (getRangeResponse.getNextStartPrimaryKey() != null) { + rangeRowQueryCriteria.setInclusiveStartPrimaryKey(getRangeResponse.getNextStartPrimaryKey()); + } else { + break; + } + } + } + + public static void main(String[] args) throws Exception { + DataDeleter dataDeleter = new DataDeleter(); + // dataDeleter.batchWriteRow(); + try { + dataDeleter.deleteDataset("UTC-20230910"); + } finally { + dataDeleter.close(); + } + + // dataDeleter.getRange("_id", "UTC-20230910", "UTC-20230911"); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java new file mode 100644 index 0000000..1b7b6b9 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataFetcher.java @@ -0,0 +1,137 @@ +package com.htfp.weather.griddata.operation; + +import com.aliyun.tablestore.grid.GridDataFetcher; +import com.aliyun.tablestore.grid.model.GridDataSet; + +import com.htfp.weather.griddata.common.TableConfigBean; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; +import ucar.ma2.Array; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/22 13:53 + * @Description : 根据维度信息获取数据 + */ +@Slf4j @Component +@DependsOn({"tableStoreConf","tableConfigBean"}) +public class GfsDataFetcher extends BaseTableOperation { + @Resource + TableConfigBean tableConfigBean; + + /** + * 获取指定变量的四维数据(time, lev, lat, lon) + * @param dataSetId 数据id + * @param variables 变量名列表 + * @param origin 维度起始索引 + * @param shape 数据形状 + * @return {@link GridDataSet} + * @throws Exception + */ + public GridDataSet queryByTableStore(String dataSetId, List variables, int[] origin, int[] shape) throws Exception { + GridDataFetcher fetcher = tableStoreGrid.getDataFetcher(tableStoreGrid.getDataSetMeta(dataSetId)); + fetcher.setVariablesToGet(variables); + fetcher.setOriginShape(origin, shape); + GridDataSet gridDataSet = fetcher.fetch(); + // Grid4D grid4D = fetcher.fetch().getVariable(variable); + return gridDataSet; + } + + /** + * 获取指定变量的二维全平面数据 + * @param variable 变量名 + * @param iLev 高度索引 + * @return {@link Array} + * @throws Exception + */ + public Array getPlane(String dataSetId, String variable, int iTime, int iLev) throws Exception { + int[] origin = new int[] {iTime, iLev, 0, 0}; + int[] shape = new int[] {1, 1, tableConfigBean.latSize, tableConfigBean.lonSize}; + GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), + origin, shape); + Array array = gridDataSet.getVariable(variable).toArray(); + System.out.println("Shape: " + array.shapeToString()); + System.out.println("Data: " + array); + return array; + } + + /** + * 获取给定变量的二维平面范围数据 + * @param variable 变量名 + * @param iLev 高度索引 + * @param iLonStart 起始经度索引 + * @param iLatLength 纬度格点数量 + * @param iLonLength 经度格点数量 + * @return {@link Array} + * @throws Exception + */ + public Array getPlane(String dataSetId, String variable, int iTime, int iLev, int iLatStart, int iLonStart, int iLatLength, int iLonLength) throws Exception { + int[] origin = new int[] {iTime, iLev, iLatStart, iLonStart}; + int[] shape = new int[] {1, 1, iLatLength, iLonLength}; + GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), + origin, shape); + Array array = gridDataSet.getVariable(variable).toArray(); + System.out.println("Shape: " + array.shapeToString()); + System.out.println("Data: " + array); + return array; + } + + /** + * 获取指定变量在某位置随高度的分布 + * @param variable 变量名 + * @param iLat 纬度索引 + * @param iLon 经度索引 + * @return {@link Array} + * @throws Exception + */ + public Array getProfile(String dataSetId, String variable, int iTime, int iLat, int iLon) throws Exception { + int[] origin = new int[] {iTime, 0, iLat, iLon}; + int[] shape = new int[] {1, tableConfigBean.levSize, 1, 1}; + GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), + origin, shape); + Array array = gridDataSet.getVariable(variable).toArray(); + return array; + } + + /** + * 获取指定变量在某经纬高处的预报序列 + * @param dataSetId 数据集ID + * @param variable 变量名 + * @param iLev 高度索引 + * @param iLat 纬度索引 + * @param iLon 经度索引 + * @throws Exception + */ + public Array getSeries(String dataSetId, String variable, int[] iTimeList, int iLev, int iLat, int iLon) throws Exception { + int[] origin = new int[] {iTimeList[0], iLev, iLat, iLon}; + int[] shape = new int[] {iTimeList.length, 1, 1, 1}; + GridDataSet gridDataSet = queryByTableStore(dataSetId, Collections.singletonList(variable), origin, shape); + Array array = gridDataSet.getVariable(variable).toArray(); + System.out.println("Shape: " + array.shapeToString()); + System.out.println("Data: " + array); + return array; + } + + /** + * 获取指定变量在某位置随时间的变化 + * @param dataSetId 数据集ID + * @param variableList 变量名列表 + * @param iTImes 时间索引数组 + * @param iLev 高度索引 + * @param iLat 纬度索引 + * @param iLon 经度索引 + * @return + * @throws Exception + */ + public GridDataSet getSeries(String dataSetId, List variableList, int[] iTImes, int iLev, int iLat, int iLon) throws Exception { + int[] origin = new int[] {iTImes[0], iLev, iLat, iLon}; + int[] shape = new int[] {iTImes.length, 1, 1, 1}; + return queryByTableStore(dataSetId, variableList, origin, shape); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java new file mode 100644 index 0000000..0856261 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/GfsDataImport.java @@ -0,0 +1,264 @@ +package com.htfp.weather.griddata.operation; + +import com.aliyun.tablestore.grid.GridDataWriter; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.StoreOptions; +import com.aliyun.tablestore.grid.model.grid.Grid2D; + +import com.htfp.weather.download.FileInfo; +import com.htfp.weather.griddata.common.GfsVariableNameEnum; +import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.info.Constant; +import com.htfp.weather.utils.MeteoUtils; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.sun.xml.internal.bind.v2.TODO; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import ucar.ma2.*; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import javax.annotation.PreDestroy; +import javax.annotation.Resource; +import java.io.File; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * @author shi_y + */ +@Slf4j +@Component +@DependsOn("tableConfigBean") +public class GfsDataImport extends BaseTableOperation { + @Resource + TableConfigBean tableConfigBean; + private final ExecutorService executorService = Executors.newFixedThreadPool(5); + /** + * init meta data to table store. + * + * @param dataSetID + * @param dataType + * @param variables + * @param shape // * @param attribute meta表的属性列 + * @return + * @throws Exception + */ + private GridDataSetMeta initMeta(String dataSetID, DataType dataType, List variables, int[] shape) throws Exception { + GridDataSetMeta meta = new GridDataSetMeta( + dataSetID, + dataType, + variables, + shape[0], + shape[1], + shape[2], + shape[3], + new StoreOptions(StoreOptions.StoreType.SLICE)); + // for (Map.Entry entry : attributes.entrySet()) { + // meta.addAttribute(entry.getKey(), entry.getValue()); + // } + meta.addAttribute("status", "INIT"); + meta.addAttribute("create_time", System.currentTimeMillis()); + tableStoreGrid.putDataSetMeta(meta); + return meta; + } + + /** + * update meta and set status to DONE when data import finished. + * + * @param meta + * @return + * @throws Exception + */ + public GridDataSetMeta updateMeta(GridDataSetMeta meta) throws Exception { + meta.addAttribute("status", "DONE"); + tableStoreGrid.updateDataSetMeta(meta); + return meta; + } + + public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception { + meta.addAttribute("status", "DONE"); + tableStoreGrid.putDataSetMeta(meta); + return meta; + } + + + public void importData(OffsetDateTime refTime) throws Exception { + log.info("[tablestore] 数据导入开始, refTime = {}...", refTime); + List fileList = getFiles(refTime); + if (CollectionUtils.isEmpty(fileList)) { + throw new AppExcpetion(ErrorCode.NO_NC_OR_GRIB_FILES); + } + String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING)); + List fileVariables = getFileVariables(tableConfigBean.variableList); + int[] shape = new int[]{tableConfigBean.timeSizeMax, tableConfigBean.levSize, tableConfigBean.latSize, tableConfigBean.lonSize}; + GridDataSetMeta meta = initMeta(dataSetId, tableConfigBean.dataType, fileVariables, shape); + List forecastHours = new ArrayList<>(); // 记录到数据库属性中 + // TODO 2024/5/13: 待优化，用于数据库的索引，必须连续，因此必须保证前一个时刻成功导入后才能导入下一个时刻，数据量大的时候导入时间较长 + + List> futures = new ArrayList<>(); + for (int i = 0; i < fileList.size(); i++) { + String file = fileList.get(i); + int iTime = i; + futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime))); + } + for (Future future : futures) { + ImportResult importResult = future.get(); + forecastHours.add(String.valueOf(importResult.getForcastHour())); + } + log.info("[tablestore] 数据导入完成, forecastHours: {}", forecastHours); + meta.setForecastHours(forecastHours); + meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli()); + putMeta(meta); + + } + + /** + * read data from netcdf file and write data to table store. + * + * @param meta 数据元信息 + * @param file netcdf文件路径 + * @param iTime 文件序号 + */ + public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime) { + log.info("[tablestore] 导入文件数据开始,_t={}: {}", iTime, file); + int time = 0; + try { + GridDataWriter writer = tableStoreGrid.getDataWriter(meta); + NetcdfFile ncFile = NetcdfFiles.open(file); + // 相对reftime的小时数，文件可能缺失，因此time可能是不连续的，但是iTime是连续的 + time = (int) ncFile.findVariable("time").read().getDouble(0); + for (String variableName : meta.getVariables()) { + if ("Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)) { + continue; + } + Variable variable = ncFile.findVariable(variableName); + + if (variable != null) { + for (int z = 0; z < meta.getzSize(); z++) { + // 注意高度索引是递增排序，即z=0对应高层低气压值 + Array array = variable.read(new int[]{0, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()}); + transferUnit(variableName, array); + Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(), + new int[]{0, 0}, new int[]{meta.getxSize(), meta.getySize()}); + writer.writeGrid2D(variable.getShortName(), iTime, z, grid2D); + } + } else { + log.warn("[tablestore] 数据文件 {} 中没有变量 {}", ncFile.getLocation(), variableName); + } + } + // 导入风速风向 + importWind(meta, ncFile, iTime); + ncFile.close(); + } catch (Exception e) { + log.error("[tablestore] 导入文件数据失败,_t={}: {}", iTime, file, e); + return new ImportResult(false, file, time, iTime); + } + log.info("[tablestore] 导入文件数据成功,_t={}: {}", iTime, file); + return new ImportResult(true, file, time, iTime); + } + + private void transferUnit(String variableName, Array array) { + if (GfsVariableNameEnum.TEMP.getNameInFile().equals(variableName)) { + MeteoUtils.kelvin2Celsius(array); + } + } + + private void importWind(GridDataSetMeta meta, NetcdfFile ncFile, int iTime) throws Exception { + // TODO 2024/5/8: 风速风向需要保存到文件中 + Variable uwnd = ncFile.findVariable("u-component_of_wind_isobaric"); + Variable vwnd = ncFile.findVariable("v-component_of_wind_isobaric"); + int xsize = meta.getxSize(); + int ysize = meta.getySize(); + int zsize = meta.getzSize(); + if (uwnd == null || vwnd == null) { + return; + } + + GridDataWriter writer = tableStoreGrid.getDataWriter(meta); + for (int z = 0; z < zsize; z++) { + Array uwndArray = uwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize}); + Array vwndArray = vwnd.read(new int[]{0, z, 0, 0}, new int[]{1, 1, xsize, ysize}); + + Array speedArray = MeteoUtils.calculateWindSpeed(uwndArray, vwndArray); + Array dirArray = MeteoUtils.calculateWindDirection(uwndArray, vwndArray); + Grid2D speedGrid2D = new Grid2D(speedArray.getDataAsByteBuffer(), speedArray.getDataType(), + new int[]{0, 0}, new int[]{xsize, ysize}); + writer.writeGrid2D("Wind_speed_isobaric", iTime, z, speedGrid2D); + + Grid2D dirGrid2D = new Grid2D(dirArray.getDataAsByteBuffer(), dirArray.getDataType(), + new int[]{0, 0}, new int[]{xsize, ysize}); + writer.writeGrid2D("Wind_direction_isobaric", iTime, z, dirGrid2D); + } + } + + private List getFileVariables(List variables) { + List fileVariables = new ArrayList<>(); + for (String variable : variables) { + String fileVariableName = GfsVariableNameEnum.getGfsVariableName(variable); + if (fileVariableName != null) { + fileVariables.add(fileVariableName); + } else { + log.warn("GFS数据文件中没有 {} 对应的变量", variable); + } + } + return fileVariables; + } + + /** + * 获取文件目录中的所有文件绝对路径 + * + * @param refTime，UTC起报时间 + * @return + */ + private List getFiles(OffsetDateTime refTime) { + String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); + File fileDir = new File(tableConfigBean.dataDir + dataFolder); + if (!fileDir.exists()) { + log.warn("文件夹 {} 不存在", fileDir); + return null; + } + List fileList = new ArrayList<>(); + String[] files = fileDir.list((dir, name) -> { + return name.startsWith("BJT") && name.endsWith("grib2"); // 判断文件名是否以“BJT”开头 + }); + if (files == null || files.length == 0) { + log.warn("文件夹 {} 中没有符合条件的文件", fileDir); + return fileList; + } + for (String filename : files) { + fileList.add(fileDir.getAbsolutePath() + File.separator + filename); + } + return fileList; + } + + @PreDestroy + public void end() { + close(); + } + + @Data + public static class ImportResult { + private boolean success; + private String file; + private int forcastHour; + private int iTime; + + public ImportResult(boolean success, String file, int forcastHour, int iTime) { + this.success = success; + this.file = file; + this.forcastHour = forcastHour; + this.iTime = iTime; + } + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java new file mode 100644 index 0000000..80b317a --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/IDataFetch.java @@ -0,0 +1,9 @@ +package com.htfp.weather.griddata.operation; + +/** + * @Author : shiyi + * @Date : 2024/2/2 13:47 + * @Description : 获取数据 + */ +public interface IDataFetch { +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java new file mode 100644 index 0000000..46c5fd2 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/QueryMeta.java @@ -0,0 +1,51 @@ +package com.htfp.weather.griddata.operation; + +import com.alicloud.openservices.tablestore.model.search.sort.FieldSort; +import com.alicloud.openservices.tablestore.model.search.sort.Sort; +import com.alicloud.openservices.tablestore.model.search.sort.SortOrder; +import com.aliyun.tablestore.grid.core.QueryBuilder; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.QueryGridDataSetResult; +import com.aliyun.tablestore.grid.model.QueryParams; +import com.htfp.weather.griddata.common.TableConfigBean; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.Collections; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/3/4 17:45 + * @Description : 查询属性表 + */ +@Component +public class QueryMeta extends BaseTableOperation { + @Resource + TableConfigBean tableConfigBean; + /** + * 查询最新的数据集信息 + * + * @return + * @throws Exception + */ + public GridDataSetMeta getLastGridDataSetMeta() throws Exception { + + QueryGridDataSetResult result = tableStoreGrid.queryDataSets( + tableConfigBean.metaIndexName, + QueryBuilder.and() + .equal("status", "DONE") + .build(), + new QueryParams(0, 10, new Sort(Collections.singletonList(new FieldSort("reference_time", SortOrder.DESC))))); + List gridDataSetMetas = result.getGridDataSetMetas(); + if (CollectionUtils.isEmpty(gridDataSetMetas)) { + throw new RuntimeException("meta table为空"); + } + return gridDataSetMetas.get(0); + } + public String queryLastReferenceTime() throws Exception { + GridDataSetMeta gridDataSetMeta = getLastGridDataSetMeta(); + return gridDataSetMeta.getGridDataSetId(); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java b/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java new file mode 100644 index 0000000..243a1e7 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/operation/UpdateTable.java @@ -0,0 +1,29 @@ +package com.htfp.weather.griddata.operation; + +import com.alicloud.openservices.tablestore.model.TableOptions; +import com.htfp.weather.griddata.common.TableConfig; + +/** + * @Author : shiyi + * @Date : 2024/2/3 17:54 + * @Description : + */ +public class UpdateTable extends BaseTableOperation { + + private void update() throws Exception { + TableOptions tableOptions = new TableOptions(TableConfig.TIME_TO_LIVE); + this.tableStoreGrid.updateStoreOption(tableOptions); + } + + public static void main(String[] args) { + TableConfig.initDatasetConfig(); + UpdateTable example = new UpdateTable(); + try { + example.update(); + } catch (Exception e) { + e.printStackTrace(); + } finally { + example.close(); + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/utils/CoordinateUtils.java b/weather-service/src/main/java/com/htfp/weather/griddata/utils/CoordinateUtils.java new file mode 100644 index 0000000..dcbbc68 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/utils/CoordinateUtils.java @@ -0,0 +1,45 @@ +package com.htfp.weather.griddata.utils; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +/** + * @Author : shiyi + * @Date : 2024/2/28 13:33 + * @Description : 物理坐标索引 + */ +public class CoordinateUtils { + public static final String DATE_TIME_PATTERN = "yyyy-MM-dd'T'HH:mmxxx"; + public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(DATE_TIME_PATTERN); + public int getLevelIndex(int lev) { + return 0; + } + + public int getLatitudeIndex(double lat) { + return 0; + } + + public int getLongitudeIndex(double lon) { + return 0; + } + + public int getTimeIndex(long time) { + return 0; + } + + /** + * 获取当前时刻对应数据库的起报时间（datasetId） + * @param timeStr + * @return + */ + public String getRefTimeInDataBase(String timeStr) { + OffsetDateTime utcTime = OffsetDateTime.parse(timeStr, DATE_TIME_FORMATTER) + .withOffsetSameInstant(ZoneOffset.ofHours(0)); + // 获取utcTime这一天对应的00 + return utcTime.withHour(2).format(DATE_TIME_FORMATTER); + + + + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/griddata/utils/GfsUtils.java b/weather-service/src/main/java/com/htfp/weather/griddata/utils/GfsUtils.java new file mode 100644 index 0000000..7729f15 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/griddata/utils/GfsUtils.java @@ -0,0 +1,39 @@ +package com.htfp.weather.griddata.utils; + +import org.apache.commons.lang3.StringUtils; +import ucar.nc2.Attribute; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; + +import java.io.IOException; +import java.time.ZonedDateTime; + +/** + * @Author : shiyi + * @Date : 2024/3/4 10:19 + * @Description : Gfs数据工具类 + */ +public class GfsUtils { + /** + * 从cdf文件中获取起报时间，并转换为时区字符串格式 + * @param ncFile + * @return 时区字符串："yyyy-MM-dd'T'HH:mmxxx"，例如"2023-09-05T08:00+08:00" + * @throws IOException + */ + public static String getRefTime(NetcdfFile ncFile) throws IOException { + Variable time = ncFile.findVariable("reftime"); + if ( time!= null) { + int hour = (int) time.read().getDouble(0); + Attribute units = time.findAttribute("units"); + if (units != null) { + String stringValue = units.getStringValue(); + if (StringUtils.isNotBlank(stringValue)) { + // "Hour since 2023-09-05T00:00:00Z" // utc时间 + ZonedDateTime zonedDateTime = ZonedDateTime.parse(stringValue.substring(11)).plusHours(hour); + return zonedDateTime.format(CoordinateUtils.DATE_TIME_FORMATTER); + } + } + } + return null; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/info/CaiYunInfo.java b/weather-service/src/main/java/com/htfp/weather/info/CaiYunInfo.java new file mode 100644 index 0000000..a0ae0ce --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/info/CaiYunInfo.java @@ -0,0 +1,81 @@ +package com.htfp.weather.info; + +import java.util.HashMap; +import java.util.Map; + +/** + * @Author : shiyi + * @Date : 2024/4/18 13:37 + * @Description : 彩云天气相关映射 + */ +public class CaiYunInfo { + private static final Map skyconMap = new HashMap<>(); + private static final Map alertNameMap = new HashMap<>(); + private static final Map alertColorMap = new HashMap<>(); + static { + // 彩云天气天气现象映射 + skyconMap.put("CLEAR_DAY", "晴（白天）"); + skyconMap.put("CLEAR_NIGHT", "晴（夜间）"); + skyconMap.put("PARTLY_CLOUDY_DAY", "多云（白天）"); + skyconMap.put("PARTLY_CLOUDY_NIGHT", "多云（夜间）"); + skyconMap.put("CLOUDY", "阴"); + skyconMap.put("LIGHT_HAZE", "轻度雾霾"); + skyconMap.put("MODERATE_HAZE", "中度雾霾"); + skyconMap.put("HEAVY_HAZE", "重度雾霾"); + skyconMap.put("LIGHT_RAIN", "小雨"); + skyconMap.put("MODERATE_RAIN", "中雨"); + skyconMap.put("HEAVY_RAIN", "大雨"); + skyconMap.put("STORM_RAIN", "暴雨"); + skyconMap.put("FOG", "雾"); + skyconMap.put("LIGHT_SNOW", "小雪"); + skyconMap.put("MODERATE_SNOW", "中雪"); + skyconMap.put("HEAVY_SNOW", "大雪"); + skyconMap.put("STORM_SNOW", "暴雪"); + skyconMap.put("DUST", "浮尘"); + skyconMap.put("SAND", "沙尘"); + skyconMap.put("WIND", "大风"); + skyconMap.put("THUNDER_SHOWER", "雷阵雨"); + skyconMap.put("HAIL", "冰雹"); + skyconMap.put("SLEET", "雨夹雪"); + + + // 彩云天气预警类型映射 + alertNameMap.put("01", "台风"); + alertNameMap.put("02", "暴雨"); + alertNameMap.put("03", "暴雪"); + alertNameMap.put("04", "寒潮"); + alertNameMap.put("05", "大风"); + alertNameMap.put("06", "沙尘暴"); + alertNameMap.put("07", "高温"); + alertNameMap.put("08", "干旱"); + alertNameMap.put("09", "雷电"); + alertNameMap.put("10", "冰雹"); + alertNameMap.put("11", "霜冻"); + alertNameMap.put("12", "大雾"); + alertNameMap.put("13", "霾"); + alertNameMap.put("14", "道路结冰"); + alertNameMap.put("15", "森林火险"); + alertNameMap.put("16", "雷雨大风"); + alertNameMap.put("17", "春季沙尘天气趋势预警"); + alertNameMap.put("18", "沙尘"); + // + alertColorMap.put("00", "白色"); + alertColorMap.put("01", "蓝色"); + alertColorMap.put("02", "黄色"); + alertColorMap.put("03", "橙色"); + alertColorMap.put("04", "红色"); + + } + + public static String getSkyConName(String skycon) { + return skyconMap.getOrDefault(skycon, "未知天气现象"); + } + + public static String getAlertName(String code) { + return alertNameMap.getOrDefault(code.substring(0,2), "未知预警类型"); + } + public static String getAlertColor(String code) { + return alertColorMap.getOrDefault(code.substring(2,4), "未知预警等级"); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/info/Constant.java b/weather-service/src/main/java/com/htfp/weather/info/Constant.java new file mode 100644 index 0000000..3941242 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/info/Constant.java @@ -0,0 +1,14 @@ +package com.htfp.weather.info; + +/** + * @Author : shiyi + * @Date : 2024/5/8 12:02 + * @Description : + */ +public class Constant { + public static final String DATA_SET_ID_FORMAT_STRING = "'UTC-'yyyyMMdd.HH"; + public static final String DATA_FOLDER_STRING = "'UTC-'yyyyMMdd.HH"; + public static final String UTC_TIME_STRING = "'UTC-'yyyyMMdd.HH"; + public static final String BJT_TIME_STRING = "'BJT-'yyyyMMdd.HH"; + public static final String API_TIME_STRING = "yyyy-MM-dd'T'HH:mmxxx"; +} diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java b/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java new file mode 100644 index 0000000..bc24dd5 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/info/GfsDownloadVariableEnum.java @@ -0,0 +1,42 @@ +package com.htfp.weather.info; + +/** + * @Author : shiyi + * @Date : 2024/4/28 16:31 + * @Description : GFS变量枚举 + */ +public enum GfsDownloadVariableEnum { + // + TEMPERATURE("TMP", "温度"), + U_WIND("UGRD", "东西风分量"), + V_WIND("VGRD", "南北风分量"), + CLOUD("TCDC", "云量"), + RELATIVE_HUMIDITY("RH", "相对湿度"), + VERTICAL_VELOCITY("DZDT", "垂直速度"), + + ; + private final String code; + private final String info; + + GfsDownloadVariableEnum(String code, String info) { + this.code = code; + this.info = info; + } + + public String getCode() { + return code; + } + + public String getInfo() { + return info; + } + + public static boolean contains(String test) { + for (GfsDownloadVariableEnum c : GfsDownloadVariableEnum.values()) { + if (c.getCode().equals(test)) { + return true; + } + } + return false; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java b/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java new file mode 100644 index 0000000..423fad0 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/info/GfsLevelsEnum.java @@ -0,0 +1,54 @@ +package com.htfp.weather.info; + +/** + * @Author : shiyi + * @Date : 2024/4/28 16:45 + * @Description : 高度层枚举 + */ +public enum GfsLevelsEnum { + // + SURFACE(9999, "地面"), + PRES_1000hPa(1000, "1000 hPa"), + PRES_975hPa(975, "975 hPa"), + PRES_950hPa(950, "950 hPa"), + PRES_925hPa(925, "925 hPa"), + PRES_900hPa(900, "900 hPa"), + PRES_850hPa(850, "850 hPa"), + PRES_800hPa(800, "800 hPa"), + PRES_750hPa(750, "750 hPa"), + PRES_700hPa(700, "700 hPa"), + PRES_650hPa(650, "650 hPa"), + PRES_600hPa(600, "600 hPa"), + PRES_550hPa(550, "550 hPa"), + PRES_500hPa(500, "500 hPa"), + PRES_450hPa(450, "450 hPa"), + PRES_400hPa(400, "400 hPa"), + PRES_350hPa(350, "350 hPa"), + + ; + + + private final Integer code; + private final String info; + + GfsLevelsEnum(Integer code, String info) { + this.code = code; + this.info = info; + } + + public Integer getCode() { + return code; + } + + public String getInfo() { + return info; + } + public static boolean contains(Integer test) { + for (GfsLevelsEnum c : GfsLevelsEnum.values()) { + if (c.getCode().equals(test)) { + return true; + } + } + return false; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java new file mode 100644 index 0000000..0260f94 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java @@ -0,0 +1,76 @@ +package com.htfp.weather.schedule; + +import com.htfp.weather.download.FileInfo; +import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.download.GfsDownloader; +import com.htfp.weather.griddata.operation.GfsDataImport; +import com.htfp.weather.info.Constant; +import com.htfp.weather.web.exception.AppExcpetion; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.checkerframework.checker.units.qual.A; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/5/8 11:36 + * @Description : 更新数据库数据，包括下载、导入、清楚本地过期数据 + */ +@Component +@Slf4j +@DependsOn({"gfsDataImport", "gfsDownloader"}) +public class GridDataProcesser { + @Resource + GfsDataImport gfsDataImport; + @Resource + GfsDownloader gfsDownloader; + @Resource + GfsDataConfig gfsDataConfig; + + public void dailyDataProcess() throws Exception { + gfsDownloader.iniTimeSetting(); + OffsetDateTime refTime = gfsDownloader.getRefTime(); + List fileInfoList = gfsDownloader.getFilesInfo(); + boolean allComplete = gfsDownloader.downloadAll(fileInfoList); + if (allComplete) { + try { + gfsDataImport.importData(refTime); + } catch (AppExcpetion e) { + log.info("导入数据失败"); + } + } + } + + public void clearExpiredData() { + // String dataFolder = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)); + File dataDir = new File(gfsDataConfig.getSaveRoot()); + if (!dataDir.exists()) { + log.warn("文件夹 {} 不存在", dataDir); + return; + } + OffsetDateTime now = OffsetDateTime.now(); + for (File subDir : dataDir.listFiles()) { + OffsetDateTime refTime = LocalDateTime.parse(subDir.getName(), DateTimeFormatter.ofPattern(Constant.DATA_FOLDER_STRING)).atOffset(ZoneOffset.UTC); + if (refTime.isBefore(now.minusDays(3))) { + try { + FileUtils.deleteDirectory(subDir); + log.info("删除过期数据文件夹成功： {}", subDir.getName()); + } catch (IOException e) { + log.info("删除过期数据文件夹失败： {}", subDir.getName()); + } + } + } + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java new file mode 100644 index 0000000..8508aee --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/DateTimeUtils.java @@ -0,0 +1,135 @@ +package com.htfp.weather.utils; + +import java.time.*; +import java.time.format.DateTimeFormatter; + +/** + * @Author : shiyi + * @Date : 2024/1/23 18:18 + * @Description : 日期时间工具类 + */ +public class DateTimeUtils { + public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mmxxx"; + public static final DateTimeFormatter DEFAULT_FORMATTER = DateTimeFormatter.ofPattern(DEFAULT_PATTERN); + /** + * 将某时区的时间字符串更改为其他时区的时间字符串, + * + * @param dateTimeStr 输入的时间字符串，需包含时区信息，如+08:00 + * @param offsetTo 输出时间的时区，如8代表东八区 + * @return 更改时区之后的时间字符串，输出格式与输入格式一致 + */ + public static String convertZonedDateTime(String dateTimeStr, int offsetTo, String formatStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr); + // 转换为UTC时区的ZonedDateTime对象，时间点保持一致 + OffsetDateTime offsetDateTime = OffsetDateTime.parse(dateTimeStr, formatter) + .withOffsetSameInstant(ZoneOffset.ofHours(offsetTo)); + return offsetDateTime.format(formatter); + } + + /** + * 以指定的格式，将某时区的时间字符串更改为其他时区的时间字符串 + * + * @param dateTimeStr 输入的时间字符串，不需要包含时区信息，如果含有时区信息将会被忽略 + * @param zoneFrom 输入时间的时区, dateTimeStr中的时区信息将被忽略 + * @param zoneTo 输出时间的时区，如8代表东八区 + * @param formatStr 输入和输出的时间字符串格式，可以不包含时区信息 + * @return 指定时区表示的时间字符串 + */ + public static String convertZonedDateTime(String dateTimeStr, int zoneFrom, int zoneTo, String formatStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr); + ZoneId zoneIdFrom = ZoneId.ofOffset("", ZoneOffset.ofHours(zoneFrom)); + ZoneId zoneIdTo = ZoneId.ofOffset("", ZoneOffset.ofHours(zoneTo)); + + // LocalDateTime解析将会忽略dateTimeStr中的时区信息 + // OffsetDateTime.parse(dateTimeStr) + ZonedDateTime zonedDateTime = LocalDateTime.parse(dateTimeStr, formatter) + .atZone(zoneIdFrom) + .withZoneSameInstant(zoneIdTo); + return zonedDateTime.format(formatter); + } + + + public static OffsetDateTime getUTCDateTime(OffsetDateTime localDateTime) { + return localDateTime.withOffsetSameInstant(ZoneOffset.UTC); + } + + // public static LocalDateTime getUTCDateTime(String timeString, String formatStr) { + // DateTimeFormatter formatter = new DateTimeFormatterBuilder() + // .appendPattern(formatStr) + // .toFormatter(); + // + // // 转换为UTC时区的ZonedDateTime对象，时间点保持一致 + // ZonedDateTime utcZonedDateTime = LocalDateTime.parse(timeString, formatter) + // .atZone(ZoneId.systemDefault()) + // .withZoneSameInstant(ZoneId.of("UTC")); + // return utcZonedDateTime.toLocalDateTime(); + // } + + public static ZonedDateTime getLocalZoneDateTime(OffsetDateTime utcDateTime) { + ZonedDateTime zonedDateTime = utcDateTime.toZonedDateTime().withZoneSameInstant(ZoneId.systemDefault()); + return zonedDateTime; + } + /** + * 将UTC时间字符串转换为将北京时时间字符串 + * + * @param utcTimeStr UTC时间字符串 + * @param formatStr 输入和输出的字符串格式 + * @return 北京时（东八区）时间字符串 + */ + public static String getBJTDateTimeStringFromUTC(String utcTimeStr, String formatStr) { + return convertZonedDateTime(utcTimeStr, 0, 8, formatStr); + } + + /** + * 将北京时时间字符串转换为UTC时间字符串 + * + * @param bjtTimeStr 北京时（东八区）时间字符串 + * @param formatStr 输入和输出的字符串格式 + * @return UTC时间字符串 + */ + public static String getUTCDateTimeStringFromBJT(String bjtTimeStr, String formatStr) { + return convertZonedDateTime(bjtTimeStr, 8, 0, formatStr); + } + + /** 获取指定日期的时间戳（毫秒） + * @param dateTimeStr 时区时间字符串 + * @param formatStr 指定日期字符串的格式 + * @return 毫秒时间戳 + */ + public static long getTimestampMilli(String dateTimeStr, String formatStr) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(formatStr); + return OffsetDateTime.parse(dateTimeStr, formatter).toInstant().toEpochMilli(); + } + public static long getTimestampMilli(String dateTimeStr) { + return OffsetDateTime.parse(dateTimeStr, DEFAULT_FORMATTER).toInstant().toEpochMilli(); + } + /** + * 将时间戳转换为时间字符串 + * @param timestampMilli + * @param zoneOffset + * @return yyyy-MM-dd'T'HH:mmxxx 格式字符串 + */ + public static String getTimeStringFromMilli(long timestampMilli, int zoneOffset) { + Instant instant = Instant.ofEpochMilli(timestampMilli); + // 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间 + ZonedDateTime zonedDateTime = instant.atZone(ZoneId.ofOffset("", ZoneOffset.ofHours(zoneOffset))); + return zonedDateTime.format(DEFAULT_FORMATTER); + } + public static String getTimeStringFromMilli(long timestampMilli, String zoneId) { + Instant instant = Instant.ofEpochMilli(timestampMilli); + // 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间 + ZonedDateTime zonedDateTime = instant.atZone(ZoneId.of(zoneId)); + return zonedDateTime.format(DEFAULT_FORMATTER); + } + /** + * 将时间戳转换为本地时区的字符串 + * @param timestampMilli + * @return yyyy-MM-dd'T'HH:mmxxx 格式字符串 + */ + public static String getTimeStringFromMilli(long timestampMilli) { + Instant instant = Instant.ofEpochMilli(timestampMilli); + // 使用 ZonedDateTime 类将 Instant 对象转换为本地时区时间 + ZonedDateTime zonedDateTime = instant.atZone(ZoneId.systemDefault()); + return zonedDateTime.format(DEFAULT_FORMATTER); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/DownloadUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/DownloadUtils.java new file mode 100644 index 0000000..5870944 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/DownloadUtils.java @@ -0,0 +1,9 @@ +package com.htfp.weather.utils; + +/** + * @Author : shiyi + * @Date : 2024/4/25 18:31 + * @Description : 文件下载https://www.jianshu.com/p/3b269082cbbb + */ +public class DownloadUtils { +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java new file mode 100644 index 0000000..7c7c03e --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/HttpClientUtils.java @@ -0,0 +1,86 @@ +package com.htfp.weather.utils; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Request.Builder; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @Author : shiyi + * @Date : 2024/1/23 16:19 + * @Description : HttpClient工具类 + */ +@Slf4j +public class HttpClientUtils { + private static final OkHttpClient OKHTTP_CLIENT = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .build(); + + // public static T sendGet(String url, Map params, Class responseClass) { + // MultiValueMap queryParams = new LinkedMultiValueMap<>(); + // if (params != null) { + // params.forEach(queryParams::add); + // } + // Mono mono = WebClient.create(url).method(HttpMethod.GET) + // .uri(uriBuilder -> uriBuilder + // .queryParams(queryParams) + // .build()).retrieve().bodyToMono(responseClass); + // T responseBody; + // try { + // responseBody = mono.block(); + // } catch (WebClientResponseException e) { + // log.error("Get 请求错误, {}", e.getMessage()); + // return null; + // } + // return responseBody; + // } + public static T sendGet(String url, Map params, Class responseClass) throws Exception { + String jsonStr = sendGet(url, params); + return JSONUtils.json2pojo(jsonStr, responseClass); + } + public static String sendGet(String url, Map params) throws IOException { + StringBuilder stringBuilder = new StringBuilder(); + if (!CollectionUtils.isEmpty(params)) { + params.keySet().forEach(res -> { + if (StringUtils.isNotBlank(stringBuilder)) { + stringBuilder.append("&"); + } else { + stringBuilder.append("?"); + } + try { + stringBuilder.append(String.format("%s=%s", res, URLEncoder.encode(params.get(res), "UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + }); + } + Request request = new Builder() + .url(url + stringBuilder) + .build(); + + try (Response response = OKHTTP_CLIENT.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("okhttp Unexpected code " + response); + } + ResponseBody body = response.body(); + if (body != null) { + return body.string(); + } else { + return null; + } + } + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/JSONUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/JSONUtils.java new file mode 100644 index 0000000..da0fd75 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/JSONUtils.java @@ -0,0 +1,92 @@ +package com.htfp.weather.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JSONUtils { + + private final static ObjectMapper objectMapper = new ObjectMapper(); + + private JSONUtils() { + + } + + public static ObjectMapper getInstance() { + + return objectMapper; + } + + /** + * javaBean,list,array convert to json string + */ + public static String obj2json(Object obj) throws Exception { + return objectMapper.writeValueAsString(obj); + } + + /** + * json string convert to javaBean + */ + public static T json2pojo(String jsonStr, Class clazz) + throws Exception { + return objectMapper.readValue(jsonStr, clazz); + } + + /** + * json string convert to map + */ + public static Map json2map(String jsonStr) + throws Exception { + return objectMapper.readValue(jsonStr, Map.class); + } + + /** + * json string convert to map with javaBean + */ + public static Map json2map(String jsonStr, Class clazz) + throws Exception { + Map> map = (Map>) objectMapper.readValue(jsonStr, + new TypeReference>() { + }); + Map result = new HashMap(); + for (Map.Entry> entry : map.entrySet()) { + result.put(entry.getKey(), map2pojo(entry.getValue(), clazz)); + } + return result; + } + + /** + * json array string convert to list with javaBean + */ + public static List json2list(String jsonArrayStr, Class clazz) + throws Exception { + List> list = (List>) objectMapper.readValue(jsonArrayStr, + new TypeReference>() { + }); + List result = new ArrayList(); + for (Map map : list) { + result.add(map2pojo(map, clazz)); + } + return result; + } + + /** + * map convert to javaBean + */ + public static T map2pojo(Map map, Class clazz) { + return objectMapper.convertValue(map, clazz); + } + + public static void pojo2jsonFile(Object obj, String filePath) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + objectMapper.writeValue(new File(filePath), obj); + } +} \ No newline at end of file diff --git a/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java new file mode 100644 index 0000000..2f34446 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/MeteoUtils.java @@ -0,0 +1,144 @@ +package com.htfp.weather.utils; + +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; +import org.checkerframework.checker.units.qual.A; +import ucar.ma2.Array; +import ucar.ma2.ArrayFloat; +import ucar.ma2.Index; +import ucar.nc2.units.DateType; + +import java.nio.ByteBuffer; +import java.text.DecimalFormat; +import java.util.Arrays; + +/** + * @Author : shiyi + * @Date : 2024/1/24 11:32 + * @Description : 气象数据处理工具类 + */ +public class MeteoUtils { + + /** + * 风速单位转换 + * + * @param mps 风速 (m/s) + * @return 风速 (km/h) + */ + public static double mPerSecond2kmPerHour(double mps) { + return mps * 3.6; + } + + /** + * @param kph 风速 (km/h) + * @return 风速 (m/s) + */ + public static double kmPerHour2mPerSecond(double kph) { + return kph / 3.6; + } + + // public static double pressure2Height(double pressure) { + // + // } + // + // public static double height2Pressure(double pressure) { + // + // } + + /** + * 开尔文转摄氏度 + */ + public static Array kelvin2Celsius(Array array) { + for (int i = 0; i < array.getSize(); i++) { + array.setFloat(i, array.getFloat(i) - 273.15f); + } + return array; + } + public static double calculateWindSpeed(double u, double v) { + return Math.sqrt(u * u + v * v); + } + // + // @Deprecated + // public static Array calculateWindSpeed(Array u, Array v) { + // // // FIXME 2024/5/13: io耗时太多，对于一般大小的矩阵不使用 + // uvValid(u, v); + // try (NDManager manager = NDManager.newBaseManager()) { + // int[] shapeArray = u.getShape(); + // Shape shape = new Shape(Arrays.stream(shapeArray).mapToLong(i -> (long) i).toArray()); + // NDArray uwnd = manager.create((float[]) u.getStorage(), shape); + // NDArray vwnd = manager.create((float[]) v.getStorage(), shape); + // NDArray windSpeed = (uwnd.square().add(vwnd.square())).sqrt(); + // ByteBuffer byteBuffer = windSpeed.toByteBuffer(); + // return Array.factory(u.getDataType(), shapeArray, byteBuffer); + // } catch (Exception e) { + // throw e; + // } + // } + + private static void uvValid(Array u, Array v) { + if (u.getSize() != v.getSize()) { + throw new IllegalArgumentException("u 和 v 数据量不一致"); + } + if (u.getSize() == 0 || v.getSize() == 0) { + throw new IllegalArgumentException("u 和 v 数据不能为空"); + } + if (!Arrays.equals(u.getShape(), v.getShape())) { + throw new IllegalArgumentException("u 和 v 数据形状不一致"); + } + if (!u.getDataType().equals(v.getDataType())) { + throw new IllegalArgumentException("u 和 v 数据类型不一致"); + } + } + + public static Array calculateWindSpeed(Array u, Array v) { + uvValid(u, v); + Array windSpeed = new ArrayFloat(u.getShape()); + for (int i = 0; i < windSpeed.getSize(); i++) { + windSpeed.setFloat(i, (float) calculateWindSpeed(u.getFloat(i), v.getFloat(i))); + } + return windSpeed; + + } + + public static Array calculateWindDirection(Array u, Array v) { + uvValid(u, v); + Array windDirection = new ArrayFloat(u.getShape()); + for (int i = 0; i < windDirection.getSize(); i++) { + windDirection.setFloat(i, (float) calculateWindDirection(u.getFloat(i), v.getFloat(i))); + } + return windDirection; + + } + public static double calculateWindDirection(double u, double v) { + // DecimalFormat df = new DecimalFormat("#.00"); + + double windDirDeg; + + if (nearToZero(u) && v > 0.) { + windDirDeg = 180; + } else if (nearToZero(u) && v < 0.) { + windDirDeg = 0; + } else if (u > 0. && nearToZero(v)) { + windDirDeg = 270; + } else if (u < 0. && nearToZero(v)) { + windDirDeg = 90; + } else if (nearToZero(u) && nearToZero(v)) { + windDirDeg = 0; + } else { + windDirDeg = calculateWindDirection0(u, v); + } + return (float) windDirDeg; + } + + public static double calculateWindDirection0(double u, double v) { + // DecimalFormat df = new DecimalFormat("#.00"); + double windDirRad = Math.atan2(v, u); + return (270 - windDirRad * 180 / Math.PI) % 360; + } + + private static boolean nearToZero(double value) { + return Math.abs(value) < 1e-5; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/NdArrayUtils.java b/weather-service/src/main/java/com/htfp/weather/utils/NdArrayUtils.java new file mode 100644 index 0000000..49a24e9 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/NdArrayUtils.java @@ -0,0 +1,42 @@ +package com.htfp.weather.utils; + +import java.util.Arrays; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/22 20:05 + * @Description : 多维数组工具类 + */ +public class NdArrayUtils { + /** + * 查找距离目标值最近的索引 + * + * @param list + * @param target + * @return + */ + public static int findNearestIndex(double[] list, double target) { + int size = list.length; + if (size == 0) { + return -1; + } + if (size == 1) { + return 0; + } + + int index = Arrays.binarySearch(list, target); + if (index >= 0) { + return index; + } + + // 插入点，即最大的小于等于目标值的元素索引 + int insertPoint = -(index + 1); + if (insertPoint > 1 && Math.abs(list[insertPoint - 1] - target) <= Math.abs(list[insertPoint] - target)) { + return insertPoint - 1; + } + return insertPoint; + } + + +} diff --git a/weather-service/src/main/java/com/htfp/weather/utils/SpringUtil.java b/weather-service/src/main/java/com/htfp/weather/utils/SpringUtil.java new file mode 100644 index 0000000..8edf43f --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/utils/SpringUtil.java @@ -0,0 +1,46 @@ +package com.htfp.weather.utils; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +/** + * @Author : shiyi + * @Date : 2024/1/5 15:51 + * @Description : Spring工具类 + */ + +@Component +public class SpringUtil implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + if(SpringUtil.applicationContext == null) { + SpringUtil.applicationContext = applicationContext; + } + } + + //获取applicationContext + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + //通过name获取 Bean + public static Object getBean(String name){ + return getApplicationContext().getBean(name); + } + + //通过class获取Bean. + public static T getBean(Class clazz){ + return getApplicationContext().getBean(clazz); + } + + //通过name,以及Clazz返回指定的Bean + public static T getBean(String name,Class clazz){ + return getApplicationContext().getBean(name, clazz); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/config/CORSFilter.java b/weather-service/src/main/java/com/htfp/weather/web/config/CORSFilter.java new file mode 100644 index 0000000..dfc5d95 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/config/CORSFilter.java @@ -0,0 +1,23 @@ +package com.htfp.weather.web.config; + + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; + +@Configuration +public class CORSFilter { + + @Bean + public CorsFilter corsFilter(){ + CorsConfiguration corsConfiguration = new CorsConfiguration(); + corsConfiguration.addAllowedHeader("*"); + corsConfiguration.addAllowedOrigin("*"); + corsConfiguration.addAllowedMethod("*"); + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/**",corsConfiguration); + return new CorsFilter(source); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java new file mode 100644 index 0000000..792fda6 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java @@ -0,0 +1,113 @@ +package com.htfp.weather.web.controller; + +import com.htfp.weather.download.FileInfo; +import com.htfp.weather.download.GfsDataConfig; +import com.htfp.weather.download.GfsDownloader; +import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.operation.GfsDataImport; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.response.Result; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; + +/** + * @Author : shiyi + * @Date : 2024/4/21 15:56 + * @Description : 配置文件管理 + */ +@CrossOrigin +@RestController +@RequestMapping("/htfp/weather/config") +public class ConfigController { + @Resource + TableConfigBean tableConfigBean; + @Resource + GfsDataConfig gfsDataConfig; + @Resource + GfsDownloader gfsDownloader; + @Resource + GfsDataImport gfsDataImport; + + private final String SECRET = "htfpweather"; + + private void validSecret(String secret) { + if (!SECRET.equals(secret)) { + throw new AppExcpetion(ErrorCode.SECRET_ERROR); + } + } + @RequestMapping("/queryDatabaseConfig") + public Result queryDatabaseConfig() { + return Result.success(tableConfigBean); + } + + @RequestMapping("/updateDatabaseConfig") + public Result updateDatabaseConfig(@RequestParam String secret, @Validated @RequestBody TableConfigBean updateConfig) { + validSecret(secret); + if (updateConfig == null) { + throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空"); + } + updateConfig.valid(); + try { + tableConfigBean.writeConfig(updateConfig); + } catch (IOException e) { + throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e); + } + return Result.success(); + } + + @RequestMapping("/queryDataSourceConfig") + public Result queryDataSourceConfig() { + return Result.success(gfsDataConfig); + } + + @RequestMapping("/updateDataSourceConfig") + public Result updateDataSourceConfig(@RequestParam String secret, @Validated @RequestBody GfsDataConfig updateConfig) { + validSecret(secret); + if (updateConfig == null) { + throw new AppExcpetion(ErrorCode.CONFIG_ERROR, "配置文件不能为空"); + } + updateConfig.valid(); + try { + gfsDataConfig.writeConfig(updateConfig); + } catch (IOException e) { + throw new AppExcpetion(ErrorCode.CONFIG_ERROR, e); + } + return Result.success(); + } + + @RequestMapping("/download") + public Result download(@RequestParam String secret) { + validSecret(secret); + try { + gfsDownloader.iniTimeSetting(); + List fileInfoList = gfsDownloader.getFilesInfo(); + gfsDownloader.downloadAll(fileInfoList); + return Result.success(fileInfoList); + } catch (Exception e) { + return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage()); + } + } + + + @RequestMapping("/import") + public Result importData(@RequestBody Map params) { + String secret = params.get("secret"); + validSecret(secret); + OffsetDateTime time = OffsetDateTime.parse(params.get("time")); + try { + gfsDataImport.importData(time); + return Result.success(); + } catch (Exception e) { + e.printStackTrace(); + return Result.error(e); + } + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ControllerExceptionAdvice.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ControllerExceptionAdvice.java new file mode 100644 index 0000000..180e017 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ControllerExceptionAdvice.java @@ -0,0 +1,35 @@ +package com.htfp.weather.web.controller; + +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.response.Result; +import org.springframework.validation.BindException; +import org.springframework.validation.ObjectError; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +/** + * @Author : shiyi + * @Date : 2024/3/14 16:28 + * @Description : 校验异常捕获 + */ +@RestControllerAdvice +public class ControllerExceptionAdvice { + + @ExceptionHandler({BindException.class}) + public Result methodArgumentNotValidExceptionHandler(BindException e) { + // 从异常对象中拿到ObjectError对象 + ObjectError objectError = e.getBindingResult().getAllErrors().get(0); + return Result.error(ErrorCode.VALIDATE_ERROR, objectError.getDefaultMessage()); + } + + // @ExceptionHandler({Exception.class}) + // public Result otherExceptionHandler(Exception e) { + // return Result.error(e); + // } + + @ExceptionHandler({AppExcpetion.class}) + public Result appExceptionHandler(AppExcpetion e) { + return Result.error(e.getErrorCode(), e.getMessage()); + } +} \ No newline at end of file diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java new file mode 100644 index 0000000..2f5fbf5 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/SurfaceWeatherController.java @@ -0,0 +1,57 @@ +package com.htfp.weather.web.controller; + +import com.htfp.weather.web.pojo.hefeng.HeFengWarningResponse; +import com.htfp.weather.web.pojo.request.Position2D; +import com.htfp.weather.web.pojo.response.NowWeatherStatus; +import com.htfp.weather.web.pojo.response.Result; +import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import com.htfp.weather.web.service.surfaceapi.CaiYunServiceImpl; +import com.htfp.weather.web.service.surfaceapi.HeFengServiceImpl; +import com.htfp.weather.web.service.IDataService; +import com.htfp.weather.web.service.surfaceapi.ISurfaceDataService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/24 16:07 + * @Description : 地面天气状态 + */ +@CrossOrigin +@RestController +@RequestMapping("/htfp/weather/surface/") +public class SurfaceWeatherController { + @Resource(name = "tablestore-gfs") + IDataService dataService; + @Resource(name = "caiyun") + ISurfaceDataService surfaceDataService; + + @PostMapping("/now") + public Result queryNowWeather(@Validated @RequestBody Position2D position2D) throws Exception { + double lat = position2D.getLatitude(); + double lon = position2D.getLongitude(); + NowWeatherStatus nowWeatherStatus = surfaceDataService.getNowSurfaceWeatherStatus(lat, lon); + return Result.success(nowWeatherStatus); + } + + @PostMapping("/forecast") + public Result querySurfaceForecastWeather(@Validated @RequestBody Position2D position2D) throws Exception { + double lat = position2D.getLatitude(); + double lon = position2D.getLongitude(); + TimeSeriesDataset forecastSeries = surfaceDataService.getForecastSeries(lat, lon); + return Result.success(forecastSeries); + } + + @PostMapping ("/warning") + public Result warning(@Validated @RequestBody Position2D position2D) throws Exception { + double lat = position2D.getLatitude(); + double lon = position2D.getLongitude(); + List warning = surfaceDataService.getSurfaceWarning(lat, lon); + return Result.success(warning); + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java new file mode 100644 index 0000000..247ab8d --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ThymeleafTest.java @@ -0,0 +1,30 @@ +package com.htfp.weather.web.controller; + +import com.htfp.weather.griddata.common.TableConfigBean; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +/** + * @Author : shiyi + * @Date : 2024/4/20 17:30 + * @Description : + */ +@Controller +public class ThymeleafTest { + @Resource + TableConfigBean tableConfigBean; + + @GetMapping("index")// 页面的url地址 + public String getindex(Model model, String secret) { + if (!"shiyi".equals(secret)){ + return "wrong"; + } + model.addAttribute("tableConfig", tableConfigBean); + return "index";// 与templates中index.html对应 + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java new file mode 100644 index 0000000..9c7e0fa --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/UpperWeatherController.java @@ -0,0 +1,65 @@ +package com.htfp.weather.web.controller; + +import com.htfp.weather.utils.DateTimeUtils; +import com.htfp.weather.web.pojo.request.PlaneRequest; +import com.htfp.weather.web.pojo.request.Position3D; +import com.htfp.weather.web.pojo.request.ProfileRequest; +import com.htfp.weather.web.pojo.response.PlaneResponse; +import com.htfp.weather.web.pojo.response.ProfileResponse; +import com.htfp.weather.web.pojo.response.Result; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import com.htfp.weather.web.service.GfsDataServiceImpl; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.time.OffsetDateTime; + +/** + * @Author : shiyi + * @Date : 2024/5/8 15:15 + * @Description : + */ +@CrossOrigin +@RestController +@RequestMapping("/htfp/weather/upper/") +public class UpperWeatherController { + + @Resource(name = "tablestore-gfs") + GfsDataServiceImpl tablestoreService; + + @RequestMapping("/profileByPressure") + public Result queryProfile(@Validated @RequestBody ProfileRequest profileRequest) { + OffsetDateTime time = OffsetDateTime.parse(profileRequest.getTime()); + OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time); + String variableName = profileRequest.getVariableName(); + double latitude = profileRequest.getLatitude(); + double longitude = profileRequest.getLongitude(); + ProfileResponse profileResponse = tablestoreService.getProfile(utcDateTime, variableName, latitude, longitude); + return Result.success(profileResponse); + } + + @PostMapping("/forecast") + public Result queryTimeSeries(@Validated @RequestBody Position3D position3D) { + double latitude = position3D.getLatitude(); + double longitude = position3D.getLongitude(); + int level = position3D.getLevel(); + TimeSeriesDataset forecastSeries = tablestoreService.getForecastSeries(latitude, longitude, level); + return Result.success(forecastSeries); + } + + @PostMapping("/plane") + public Result queryPlane(@Validated @RequestBody PlaneRequest planeRequest) { + planeRequest.valid(); + String variableName = planeRequest.getVariableName(); + OffsetDateTime time = OffsetDateTime.parse(planeRequest.getTime()); + OffsetDateTime utcDateTime = DateTimeUtils.getUTCDateTime(time); + int level = planeRequest.getLevel(); + double minLat = planeRequest.getMinLat(); + double maxLat = planeRequest.getMaxLat(); + double minLon = planeRequest.getMinLon(); + double maxLon = planeRequest.getMaxLon(); + PlaneResponse forecastSeries = tablestoreService.getPlane(utcDateTime, variableName, level, minLat, maxLat, minLon, maxLon); + return Result.success(forecastSeries); + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/AppExcpetion.java b/weather-service/src/main/java/com/htfp/weather/web/exception/AppExcpetion.java new file mode 100644 index 0000000..f27551c --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/exception/AppExcpetion.java @@ -0,0 +1,25 @@ +package com.htfp.weather.web.exception; + +/** + * @Author : shiyi + * @Date : 2024/4/16 17:21 + * @Description : 自定义异常 + */ +public class AppExcpetion extends RuntimeException{ + ErrorCode errorCode; + + public ErrorCode getErrorCode() { + return errorCode; + } + public AppExcpetion(ErrorCode errorCode) { + this.errorCode = errorCode; + } + public AppExcpetion(ErrorCode errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + public AppExcpetion(ErrorCode errorCode, Throwable e) { + super(e); + this.errorCode = errorCode; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java new file mode 100644 index 0000000..c762803 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java @@ -0,0 +1,45 @@ +package com.htfp.weather.web.exception; + +/** + * @Author : shiyi + * @Date : 2024/3/14 16:30 + * @Description : 错误码枚举 + */ + +public enum ErrorCode { + // + VALIDATE_ERROR(1001, "参数校验错误"), + CONFIG_ERROR(1002, "配置相关错误"), + SECRET_ERROR(1003, "无权限访问"), + DOWNLOAD_START_ERROR(1004, "数据下载启动错误"), + HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"), + HE_FENG_REQUEST_ERROR(3002, "查询请求错误"), + CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"), + CAI_YUN_DATA_PARSE_ERROR(4003, "数据解析错误"), + + IMPORT_DATA_INITIAL_FAILED(5001, "导入数据初始化失败"), + NO_NC_OR_GRIB_FILES(5002, "文件夹不存在或中没有对应的气象数据文件"), + + DATA_SET_EMPTY(6000, "TableStore数据库为空，无法查询"), + QUERY_TIME_ERROR(6001, "目标时间不在当前数据范围内"), + LONGITUDE_INDEX_ERROR(6002, "经度坐标索引"), + LATITUDE_INDEX_ERROR(6003, "纬度坐标索引"), + LEVEL_INDEX_ERROR(6004, "高度坐标索引"), + + ; + private final int code; + private final String msg; + + ErrorCode(int code, String msg) { + this.code = code; + this.msg = msg; + } + + public int getCode() { + return code; + } + + public String getMsg() { + return msg; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunBaseResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunBaseResponse.java new file mode 100644 index 0000000..fe249e1 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunBaseResponse.java @@ -0,0 +1,44 @@ +package com.htfp.weather.web.pojo.caiyun; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/4/15 18:52 + * @Description : + */ +@Data +public class CaiYunBaseResponse { + @JsonProperty("status") + private String status; + + @JsonProperty("api_version") + private String apiVersion; + + @JsonProperty("api_status") + private String apiStatus; + + @JsonProperty("lang") + private String lang; + + @JsonProperty("unit") + private String unit; + + @JsonProperty("tzshift") + private int tzshift; + + @JsonProperty("timezone") + private String timeZone; + + @JsonProperty("server_time") + private long serverTime; + + @JsonProperty("location") + private List location; + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunForecastResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunForecastResponse.java new file mode 100644 index 0000000..986a00f --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunForecastResponse.java @@ -0,0 +1,56 @@ +package com.htfp.weather.web.pojo.caiyun; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/4/15 18:52 + * @Description : + */ +@Data +public class CaiYunForecastResponse extends CaiYunBaseResponse{ + + @JsonProperty("result") + private HourlyResult hourlyResult; + + // Getters and Setters + + @Data + public static class HourlyResult { + private Hourly hourly; + private int primary; + @JsonProperty("forecast_keypoint") + private String forecastKeypoint; + + @Data @JsonIgnoreProperties(value = {"skycon", "apparent_temperature","dswrf", "air_quality"}, ignoreUnknown = true) + public static class Hourly { + private String status; + private String description; + private List precipitation; + private List temperature; + private List wind; + private List humidity; + private List cloudrate; + private List pressure; + private List visibility; + } + @Data @JsonIgnoreProperties(value = {"probability"}, ignoreUnknown = true) + public static class BaseData { + private String datetime; + private Float value; + } + + @Data + public static class Wind { + private String datetime; + private Float speed; + private Float direction; + } + + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunNowResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunNowResponse.java new file mode 100644 index 0000000..eb102c1 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunNowResponse.java @@ -0,0 +1,63 @@ +package com.htfp.weather.web.pojo.caiyun; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @Author : shiyi + * @Date : 2024/4/15 18:52 + * @Description : + */ +@Data +public class CaiYunNowResponse extends CaiYunBaseResponse{ + + @JsonProperty("result") + private NowResult nowResult; + + // Getters and Setters + + @Data + public static class NowResult { + private RealTime realtime; + private int primary; + + @Data @JsonIgnoreProperties(value = {"apparent_temperature","dswrf", "air_quality", "life_index"}, ignoreUnknown = true) + public static class RealTime { + private String status; + private String description; + private Precipitation precipitation; + private Float temperature; + private Wind wind; + private Float humidity; + private Float cloudrate; + private Float pressure; + private Float visibility; + private String skycon; + } + @Data + public static class Wind { + private Float speed; + private Float direction; + } + + @Data + public static class Precipitation { + private Local local; + private Nearest nearest; + @Data + public static class Local { + private String status; + private String datasource; + private Float intensity; + } + @Data + public static class Nearest { + private String status; + private Float distance; + private Float intensity; + } + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunWarningResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunWarningResponse.java new file mode 100644 index 0000000..22405cd --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/caiyun/CaiYunWarningResponse.java @@ -0,0 +1,49 @@ +package com.htfp.weather.web.pojo.caiyun; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/4/15 18:52 + * @Description : + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class CaiYunWarningResponse extends CaiYunBaseResponse { + @JsonProperty("result") + private AlertResult result; + + @Data @JsonIgnoreProperties(value = {"realtime"}, ignoreUnknown = true) + public static class AlertResult{ + Alert alert; + @Data @JsonIgnoreProperties(value = {"abcodes"}, ignoreUnknown = true) + public static class Alert { + String status; + List content; + @Data + public static class AlertContent { + private String province; + private String status; + private String code; + private String description; + private String regionId; + private String county; + private Long pubtimestamp; + private Float[] latlon; + private String city; + private String alertId; + private String title; + private String adcode; + private String source; + private String location; + @JsonProperty("request_status") + private String requestStatus; + } + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/BaseHeFengData.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/BaseHeFengData.java new file mode 100644 index 0000000..741ab70 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/BaseHeFengData.java @@ -0,0 +1,24 @@ +package com.htfp.weather.web.pojo.hefeng; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/1/24 15:23 + * @Description : + */ +@Data +public class BaseHeFengData { + Float temp; // 温度; // 温度 + String icon; // 天气状况和图标的代码，图标可通过"icon"字段来调用 + String text; // 天气状况的文字描述，包括阴晴雨雪等天气状态的描述 + Float wind360; // 风向360角度 + String windDir; // 风向 + String windScale; // 风力等级 + Float windSpeed; // 风速，公里/小时 + Float humidity; // 相对湿度 + Float precip; // 当前小时累计降水量，默认单位：毫米 + Float pressure; // 大气压强，默认单位：百帕 + Float cloud; // 云量 + Float dew; // 露点温度 +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengBaseResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengBaseResponse.java new file mode 100644 index 0000000..74f0e37 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengBaseResponse.java @@ -0,0 +1,14 @@ +package com.htfp.weather.web.pojo.hefeng; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/4/17 15:04 + * @Description : + */ +@Data +public class HeFengBaseResponse { + String code; + String updateTime; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengForecastResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengForecastResponse.java new file mode 100644 index 0000000..ac897d0 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengForecastResponse.java @@ -0,0 +1,26 @@ +package com.htfp.weather.web.pojo.hefeng; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/23 18:01 + * @Description : 和风天气预报响应 + */ +@EqualsAndHashCode(callSuper = true) +@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true) +public class HeFengForecastResponse extends HeFengBaseResponse { + + List hourly; + + @Data @EqualsAndHashCode(callSuper = true) + public static class HeFengForecastHour extends BaseHeFengData { + String fxTime; // 预报时间 + + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengNowResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengNowResponse.java new file mode 100644 index 0000000..a1950d2 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengNowResponse.java @@ -0,0 +1,21 @@ +package com.htfp.weather.web.pojo.hefeng; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @Author : shiyi + * @Date : 2024/1/23 18:00 + * @Description : 和风实时天气响应 + */ +@EqualsAndHashCode(callSuper = true) +@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true) +public class HeFengNowResponse extends HeFengBaseResponse{ + HeFengNow now; + @Data @EqualsAndHashCode(callSuper = true) + public static class HeFengNow extends BaseHeFengData { + String obsTime; // 观测时间 + } + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengWarningResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengWarningResponse.java new file mode 100644 index 0000000..f0ee5eb --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/hefeng/HeFengWarningResponse.java @@ -0,0 +1,38 @@ +package com.htfp.weather.web.pojo.hefeng; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/23 18:00 + * @Description : 和风实时天气响应 + */ +@EqualsAndHashCode(callSuper = true) +@Data @JsonIgnoreProperties(value = {"fxLink", "refer"}, ignoreUnknown = true) +public class HeFengWarningResponse extends HeFengBaseResponse{ + List warning; + + @Data + public static class HeFengWarning{ + private String id; + private String sender; + private String pubTime; + private String title; + private String startTime; + private String endTime; + private String status; + private String level; + private String severity; + private String severityColor; + private String type; + private String typeName; + private String urgency; + private String certainty; + private String text; + private String related; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java new file mode 100644 index 0000000..315fc08 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ConfigUpdate.java @@ -0,0 +1,14 @@ +package com.htfp.weather.web.pojo.request; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/4/21 18:34 + * @Description : + */ +@Data +public class ConfigUpdate { + String secret; + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ForecastRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ForecastRequest.java new file mode 100644 index 0000000..2098621 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ForecastRequest.java @@ -0,0 +1,33 @@ +package com.htfp.weather.web.pojo.request; + +import lombok.Data; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * @Author : shiyi + * @Date : 2024/3/14 18:01 + * @Description : 预报请求参数 + */ +@Data +public class ForecastRequest { + + @NotNull(message = "变量名不能为空") + String variableName; + + @NotNull(message = "高度层次不能为空") + Integer level; + + @NotNull(message = "纬度不能为空") + @Min(value = -90, message = "纬度最小值为-90（南纬90°）") + @Max(value = 90, message = "纬度最大值为90（北纬90°）") + Double latitude; + + @NotNull(message = "经度不能为空") + @Min(value = -180, message = "经度最小值为-180（西经180°）") + @Max(value = 180, message = "经度最大值为180（东经180°）") + Double longitude; + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java new file mode 100644 index 0000000..15a759d --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/PlaneRequest.java @@ -0,0 +1,66 @@ +package com.htfp.weather.web.pojo.request; + +import com.htfp.weather.griddata.common.GfsVariableNameEnum; +import com.htfp.weather.info.Constant; +import com.htfp.weather.info.GfsDownloadVariableEnum; +import com.htfp.weather.info.GfsLevelsEnum; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.valid.DateTimeStr; +import com.htfp.weather.web.valid.EnumValid; +import lombok.Data; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.util.ArrayList; + +/** + * @Author : shiyi + * @Date : 2024/5/15 16:11 + * @Description : 平面数据请求体 + */ +@Data +public class PlaneRequest { + @NotNull(message = "时间不能为空") + @DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx") + String time; + + @NotNull(message = "变量名不能为空") + @EnumValid(enumClass = GfsVariableNameEnum.class, usedField = "getNameInApi", message = "变量不存在") + String variableName; + + @NotNull(message = "高度层不能为空") + @EnumValid(enumClass = GfsLevelsEnum.class, usedField = "getCode", message = "高度层不存在") + Integer level; + + @NotNull + @Min(value = 70, message = "经度最小值为70（东经180°）") + @Max(value = 140, message = "经度最大值为140（东经140°）") + private Double minLon; + + @NotNull + @Min(value = 70, message = "经度最小值为70（东经180°）") + @Max(value = 140, message = "经度最大值为140（东经140°）") + private Double maxLon; + + @NotNull + @Min(value = 0, message = "纬度最小值为0（0°）") + @Max(value = 55, message = "纬度最大值为55（北纬55°）") + private Double minLat; + + @NotNull + @Min(value = 0, message = "纬度最小值为0（0°）") + @Max(value = 55, message = "纬度最大值为55（北纬55°）") + private Double maxLat; + + public void valid() { + // TODO 2024/4/21: 校验 + if (minLon >= maxLon) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小经度必须小于最大经度"); + } + if (minLat >= maxLat) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, "最小纬度必须小于最大纬度度"); + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position2D.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position2D.java new file mode 100644 index 0000000..958b41c --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position2D.java @@ -0,0 +1,26 @@ +package com.htfp.weather.web.pojo.request; + +import lombok.Data; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * @Author : shiyi + * @Date : 2024/1/24 16:11 + * @Description : 地面位置参数 + */ +@Data +public class Position2D { + @NotNull(message = "纬度 (latitude) 不能为空") + @Min(value = -90, message = "纬度 (latitude) 最小值为-90（南纬90°）") + @Max(value = 90, message = "纬度 (latitude) 最大值为90（北纬90°）") + Double latitude; + + @NotNull(message = "经度 (longitude) 不能为空") + @Min(value = -180, message = "经度 (longitude) 最小值为-180（西经180°）") + @Max(value = 180, message = "经度 (longitude) 最大值为180（东经180°）") + Double longitude; + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position3D.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position3D.java new file mode 100644 index 0000000..6d278d7 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/Position3D.java @@ -0,0 +1,20 @@ +package com.htfp.weather.web.pojo.request; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @Author : shiyi + * @Date : 2024/1/24 16:11 + * @Description : 地面位置参数 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class Position3D extends Position2D { + @NotNull(message = "高度层次 (level) 不能为空") + Integer level; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java new file mode 100644 index 0000000..d1f7d4b --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/request/ProfileRequest.java @@ -0,0 +1,34 @@ +package com.htfp.weather.web.pojo.request; + +import com.htfp.weather.info.Constant; +import com.htfp.weather.web.valid.DateTimeStr; +import lombok.Data; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * @Author : shiyi + * @Date : 2024/5/10 17:27 + * @Description : 廓线数据请求 + */ +@Data +public class ProfileRequest { + @NotNull(message = "时间不能为空") + @DateTimeStr(format = Constant.API_TIME_STRING, message = "时间格式错误, 应为" + "yyyy-MM-ddTHH:mmxxx") + String time; + + @NotNull(message = "变量名不能为空") + String variableName; + + @NotNull(message = "纬度 (latitude) 不能为空") + @Min(value = -90, message = "纬度 (latitude) 最小值为-90（南纬90°）") + @Max(value = 90, message = "纬度 (latitude) 最大值为90（北纬90°）") + double latitude; + + @NotNull(message = "经度 (longitude) 不能为空") + @Min(value = -180, message = "经度 (longitude) 最小值为-180（西经180°）") + @Max(value = 180, message = "经度 (longitude) 最大值为180（东经180°）") + double longitude; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java new file mode 100644 index 0000000..dc84448 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/NowWeatherStatus.java @@ -0,0 +1,23 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/1/24 11:28 + * @Description : 当前天气状况 + */ +@Data +public class NowWeatherStatus { + String time; // 数据时间，北京时 + String weatherText; // 天气状况 + float temp; // 温度; // 温度 + float windSpeed; // 风速，m/s + float wind360; // 风向360角度 + // String windDir; // 风向, 文字描述，如“西北风” + // String windScale; // 风力等级 + float humidity; // 相对湿度 % + // float pressure; // 大气压强，默认单位：百帕 + float cloud; // 云量 + float precip; //降水量 +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/PlaneResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/PlaneResponse.java new file mode 100644 index 0000000..d531edd --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/PlaneResponse.java @@ -0,0 +1,15 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/5/15 16:11 + * @Description : 平面数据 + */ +@Data +public class PlaneResponse { + double[] latList; + double[] lonList; + float[][] values; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileResponse.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileResponse.java new file mode 100644 index 0000000..844a810 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/ProfileResponse.java @@ -0,0 +1,21 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/5/8 15:19 + * @Description : 高度廓线 + */ +@Data +public class ProfileResponse { + int[] pressureLevels; + // int[ pressureHeight; + float[] values; + + public ProfileResponse(int[] pressureLevels, float[] values) { + this.pressureLevels = pressureLevels; + // this.pressureHeight = pressureHeight; + this.values = values; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java new file mode 100644 index 0000000..63ef27c --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/Result.java @@ -0,0 +1,46 @@ +package com.htfp.weather.web.pojo.response; + +import com.htfp.weather.web.exception.ErrorCode; +import lombok.Data; +import lombok.ToString; + +/** + * @Author : shiyi + * @Date : 2024/1/24 16:16 + * @Description : 统一返回结构 + */ +@Data +@ToString +public class Result { + private boolean success; + private Integer code; + private String message; + private Object data; + + + public Result(boolean success, Integer code, String msg, Object data) { + this.success = success; + this.code = code; + this.message = msg; + this.data = data; + } + + public static Result success(Object data) { + return new Result(true, 200, "success", data); + } + + public static Result success() { + return new Result(true, 200, "success", null); + } + + public static Result error(ErrorCode error) { + return new Result(false, error.getCode(), error.getMsg(), null); + } + public static Result error(ErrorCode error, String msg) { + return new Result(false, error.getCode(), error.getMsg() + msg, null); + } + + public static Result error(Throwable cause) { + return new Result(false, 1001, cause.getMessage(), null); + } +} \ No newline at end of file diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/SurfaceWeatherWarning.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/SurfaceWeatherWarning.java new file mode 100644 index 0000000..846f0f7 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/SurfaceWeatherWarning.java @@ -0,0 +1,19 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/4/17 15:44 + * @Description : 地面天气预警信息 + */ +@Data +public class SurfaceWeatherWarning { + String pubTime; + // String startTime; + String title; + String text; + String typeName; + String severityColor; + String source; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeries.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeries.java new file mode 100644 index 0000000..5999325 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeries.java @@ -0,0 +1,17 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/24 13:38 + * @Description : 时间序列 + */ +@Data +public class TimeSeries { + String name; + List time; + List values; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java new file mode 100644 index 0000000..a79954a --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/pojo/response/TimeSeriesDataset.java @@ -0,0 +1,37 @@ +package com.htfp.weather.web.pojo.response; + +import lombok.Data; + +import java.sql.Time; +import java.util.ArrayList; +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/1/24 13:46 + * @Description : 将和风的逐小时数据集转换为变量时间序列数据集 + */ +@Data +public class TimeSeriesDataset { + List time; + float[] temp; + float[] windSpeed; + float[] wind360; + // List windScale; + float[] humidity; + // float[] pressure; + float[] cloud; + float[] precip; + + public TimeSeriesDataset(int size) { + time = new ArrayList<>(size); + temp = new float[size]; + windSpeed = new float[size]; + wind360 = new float[size]; + // windScale = new float[size; + humidity = new float[size]; + // pressure = new float[size; + cloud = new float[size]; + precip = new float[size]; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java new file mode 100644 index 0000000..dc5d552 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/GfsDataServiceImpl.java @@ -0,0 +1,249 @@ +package com.htfp.weather.web.service; + +import com.aliyun.tablestore.grid.consts.AttributionEnum; +import com.aliyun.tablestore.grid.model.GridDataSet; +import com.aliyun.tablestore.grid.model.GridDataSetMeta; +import com.aliyun.tablestore.grid.model.grid.Grid4D; +import com.htfp.weather.griddata.common.GfsVariableNameEnum; +import com.htfp.weather.griddata.common.TableConfigBean; +import com.htfp.weather.griddata.operation.GfsDataFetcher; +import com.htfp.weather.griddata.operation.QueryMeta; +import com.htfp.weather.info.Constant; +import com.htfp.weather.utils.DateTimeUtils; +import com.htfp.weather.utils.NdArrayUtils; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.response.PlaneResponse; +import com.htfp.weather.web.pojo.response.ProfileResponse; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import ucar.ma2.Array; + +import javax.annotation.Resource; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @Author : shiyi + * @Date : 2024/2/2 15:47 + * @Description : 获取tablestore格点数据 + */ +@Service("tablestore-gfs") @Slf4j +public class GfsDataServiceImpl implements IDataService { + @Resource + GfsDataFetcher gfsDataFetcher; + @Resource + QueryMeta queryMeta; + @Resource + TableConfigBean tableConfigBean; + + + public ProfileResponse getProfile(OffsetDateTime targetTime, String variableName, double latitude, double longitude) { + String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName); + + if (targetVariable == null) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在"); + } + + GridDataSetMeta lastGridDataSetMeta = null; + try { + lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()"); + } + int iTime = getTimeIndex(targetTime, lastGridDataSetMeta); + int iLat = getLatitudeIndex(latitude); + int iLon = getLongitudeIndex(longitude); + try { + String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); + int[] pressureLevels = tableConfigBean.getLevList(); + Array array = gfsDataFetcher.getProfile(gridDataSetId, targetVariable, iTime, iLat, iLon); + float[] values = (float[]) array.getStorage(); + return new ProfileResponse(pressureLevels, values); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + public TimeSeriesDataset getForecastSeries(double latitude, double longitude, int pressureLevel) { + GridDataSetMeta lastGridDataSetMeta = null; + try { + lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()"); + } + long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName()); + OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0)); + OffsetDateTime currentTime = DateTimeUtils.getUTCDateTime(OffsetDateTime.now()); + if (currentTime.isBefore(refTime)) { + throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR); + } + int startHour = getForecastHourFromTargetTime(currentTime, refTime); + List forecastHours = lastGridDataSetMeta.getForecastHours(); + List iTimeList = new ArrayList<>(); + List timeList = new ArrayList<>(); + + for (int i = 0; i < forecastHours.size(); i++) { + int hour = Integer.parseInt(forecastHours.get(i)); + if (hour >= startHour) { + iTimeList.add(i); + String timeStr = DateTimeUtils.getLocalZoneDateTime(refTime.plusHours(hour)) + .format(DateTimeFormatter.ofPattern(Constant.API_TIME_STRING)); + timeList.add(timeStr); + } + if (iTimeList.size() >= 24) { + break; + } + } + int[] iTimes = iTimeList.stream().mapToInt(Integer::valueOf).toArray(); + int iLat = getLatitudeIndex(latitude); + int iLon = getLongitudeIndex(longitude); + int iLev = getLevelIndex(pressureLevel); + List variableNames = GfsVariableNameEnum.getVariableNamesInFile(); + try { + String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); + GridDataSet gridDataSet = gfsDataFetcher.getSeries(gridDataSetId, variableNames, iTimes, iLev, iLat, iLon); + return buildTimeSeriesDataset(timeList, gridDataSet); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public PlaneResponse getPlane(OffsetDateTime targetTime, String variableName, int level, double minLat, double maxLat, double minLon, double maxLon) { + String targetVariable = GfsVariableNameEnum.getGfsVariableName(variableName); + + if (targetVariable == null) { + throw new AppExcpetion(ErrorCode.VALIDATE_ERROR, variableName + "变量在数据库中不存在"); + } + GridDataSetMeta lastGridDataSetMeta = null; + try { + lastGridDataSetMeta = queryMeta.getLastGridDataSetMeta(); + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.DATA_SET_EMPTY, ": e.getMessage()"); + } + + int iTime = getTimeIndex(targetTime, lastGridDataSetMeta); + int iLev = getLevelIndex(level); + int iMinLat = getLatitudeIndex(minLat); + int iMaxLat = getLatitudeIndex(maxLat); + int iMinLon = getLongitudeIndex(minLon); + int iMaxLon = getLongitudeIndex(maxLon); + int latLength = iMaxLat - iMinLat + 1; + int lonLength = iMaxLon - iMinLon + 1; + + try { + String gridDataSetId = lastGridDataSetMeta.getGridDataSetId(); + Array array = gfsDataFetcher.getPlane(gridDataSetId, targetVariable, iTime, iLev, iMinLat, iMinLon, latLength, lonLength); + + return buildPlane(array, iMinLat, iMinLon, latLength, lonLength); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private PlaneResponse buildPlane(Array array, int iMinLat, int iMinLon, int latLength, int lonLength) { + PlaneResponse planeResponse = new PlaneResponse(); + float[][] values = (float[][]) array.reduce().copyToNDJavaArray(); + planeResponse.setLatList(Arrays.copyOfRange(tableConfigBean.getLatList(), iMinLat, iMinLat + latLength)); + planeResponse.setLonList(Arrays.copyOfRange(tableConfigBean.getLonList(), iMinLon, iMinLon + lonLength)); + planeResponse.setValues(values); + return planeResponse; + } + + private TimeSeriesDataset buildTimeSeriesDataset(List timeList, GridDataSet gridDataSet) { + TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(timeList.size()); + timeSeriesDataset.setTime(timeList); + Grid4D grid4D; + grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.TEMP)); + if (grid4D != null) { + timeSeriesDataset.setTemp((float[]) grid4D.toArray().getStorage()); + } + grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND_SPEED)); + if (grid4D != null) { + timeSeriesDataset.setWindSpeed((float[]) grid4D.toArray().getStorage()); + } + grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.WIND360)); + if (grid4D != null) { + timeSeriesDataset.setWind360((float[]) grid4D.toArray().getStorage()); + } + grid4D = gridDataSet.getVariable(GfsVariableNameEnum.getGfsVariableName(GfsVariableNameEnum.CLOUD)); + if (grid4D != null) { + timeSeriesDataset.setCloud((float[]) grid4D.toArray().getStorage()); + } + return timeSeriesDataset; + } + + private int getTimeIndex(OffsetDateTime targetTime, GridDataSetMeta lastGridDataSetMeta) { + long milli = (long) lastGridDataSetMeta.getAttributes().get(AttributionEnum.REFERENCE_TIME.getName()); + OffsetDateTime refTime = OffsetDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneOffset.ofHours(0)); + if (targetTime.isBefore(refTime)) { + throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR); + } + + List forecastHours = lastGridDataSetMeta.getForecastHours(); + String targetHour = String.valueOf(getForecastHourFromTargetTime(targetTime, refTime)); + if (!forecastHours.contains(targetHour)) { + throw new AppExcpetion(ErrorCode.QUERY_TIME_ERROR); + } + int iTime = forecastHours.indexOf(targetHour); + return iTime; + } + + private int getLongitudeIndex(double lon) { + double[] lonList = tableConfigBean.getLonList(); + if (lonList.length == 0) { + throw new AppExcpetion(ErrorCode.LONGITUDE_INDEX_ERROR, "经度列表为空， 无法查找索引"); + } + double minLon = lonList[0]; + double maxLon = lonList[lonList.length - 1]; + if (minLon > lon || lon > maxLon) { + throw new AppExcpetion(ErrorCode.LONGITUDE_INDEX_ERROR, "经度不在有效范围内: " + minLon + " ~ " + maxLon); + } + int nearestIndex = NdArrayUtils.findNearestIndex(lonList, lon); + if (nearestIndex == -1) { + throw new AppExcpetion(ErrorCode.LONGITUDE_INDEX_ERROR, "找不到" + lon + "的经度索引"); + } + return nearestIndex; + } + + private int getLatitudeIndex(double lat) { + double[] latList = tableConfigBean.getLatList(); + if (latList.length == 0) { + throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "纬度列表为空， 无法查找索引"); + } + double minLat = latList[0]; + double maxLat = latList[latList.length - 1]; + + if (minLat > lat || lat > maxLat) { + throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "纬度不在有效范围内: " + minLat + " ~ " + maxLat); + } + int nearestIndex = NdArrayUtils.findNearestIndex(latList, lat); + if (nearestIndex == -1) { + throw new AppExcpetion(ErrorCode.LATITUDE_INDEX_ERROR, "找不到" + lat + "的纬度索引"); + } + return nearestIndex; + } + + private int getLevelIndex(int pressureLevel) { + List levList = Arrays.stream(tableConfigBean.getLevList()).boxed().collect(Collectors.toList()); + if (CollectionUtils.isEmpty(levList)) { + throw new AppExcpetion(ErrorCode.LEVEL_INDEX_ERROR, "经度列表为空， 无法查找索引"); + } + return levList.indexOf(pressureLevel); + } + + private int getForecastHourFromTargetTime(OffsetDateTime targetTime, OffsetDateTime refTime) { + return (int) Duration.between(refTime, targetTime).toHours(); + } + + +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/IDataService.java b/weather-service/src/main/java/com/htfp/weather/web/service/IDataService.java new file mode 100644 index 0000000..50e1122 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/IDataService.java @@ -0,0 +1,9 @@ +package com.htfp.weather.web.service; + +/** + * @Author : shiyi + * @Date : 2024/3/15 10:51 + * @Description : 数据调用接口 + */ +public interface IDataService { +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java new file mode 100644 index 0000000..c4cdf66 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImpl.java @@ -0,0 +1,172 @@ +package com.htfp.weather.web.service.surfaceapi; + +import com.htfp.weather.info.CaiYunInfo; +import com.htfp.weather.utils.DateTimeUtils; +import com.htfp.weather.utils.MeteoUtils; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.caiyun.*; +import com.htfp.weather.web.pojo.caiyun.CaiYunNowResponse.NowResult.RealTime; +import com.htfp.weather.web.pojo.caiyun.CaiYunForecastResponse.HourlyResult.Hourly; +import com.htfp.weather.web.pojo.caiyun.CaiYunWarningResponse.AlertResult.Alert.AlertContent; +import com.htfp.weather.web.pojo.response.NowWeatherStatus; +import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static com.htfp.weather.utils.HttpClientUtils.sendGet; + +/** + * @Author : shiyi + * @Date : 2024/4/15 18:42 + * @Description : 彩云天气 + */ +@Service("caiyun") @Slf4j +public class CaiYunServiceImpl implements ISurfaceDataService { + + private final String key = "Tc9tgOYr5jlPPlEw"; + private final HashMap variableNameMap = + new HashMap() {{ + }}; + static final String STATUS_OK = "ok"; + public T caiYunRequest(String url, HashMap params, Class responseClass) { + params.put("units", "metric:v2"); + try { + T response = sendGet(url, params, responseClass); + if (response != null && STATUS_OK.equals(response.getStatus())) { + return response; + } else if (response == null) { + log.error("[彩云天气] 请求失败"); + throw new AppExcpetion(ErrorCode.CAI_YUN_REQUEST_ERROR); + } else { + log.error("[彩云天气] {}: {}", ErrorCode.CAI_YUN_REQUEST_ERROR.getMsg(), response); + throw new AppExcpetion(ErrorCode.CAI_YUN_REQUEST_ERROR); + } + } catch (Exception e) { + log.error("[彩云天气] 请求结果处理错误, {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + /** + * 获取地面指定位置的天气状况 + * + * @throws Exception + */ + @Override + public NowWeatherStatus getNowSurfaceWeatherStatus(double lat, double lon) throws Exception { + CaiYunNowResponse caiYunNowWeather = getCaiYunNowWeather(lat, lon); + return buildNowSurfaceWeatherStatus(caiYunNowWeather); + } + + + /** + * 获取地面指定位置某变量的24小时预报序列 + * + * @throws Exception + */ + @Override + public TimeSeriesDataset getForecastSeries(double lat, double lon) throws Exception { + Hourly hourly = getCaiYunForecast24(lat, lon); + return buildForecastSeries(hourly); + } + + /** + * 获取地面指定位置的气象预警信息 + * + * @throws Exception + */ + @Override + public List getSurfaceWarning(double lat, double lon) throws Exception { + List caiYunWarnings = getCaiYunWarning(lat, lon); + return buildSurfaceWarning(caiYunWarnings); + } + + public CaiYunNowResponse getCaiYunNowWeather(double lat, double lon) throws Exception { + String url = String.format("https://api.caiyunapp.com/v2.6/%s/%.2f,%.2f/realtime", key, lon, lat); + HashMap params = new HashMap<>(); + return caiYunRequest(url, params, CaiYunNowResponse.class); + } + + private Hourly getCaiYunForecast24(double lat, double lon) { + String url = String.format("https://api.caiyunapp.com/v2.6/%s/%.2f,%.2f/hourly", key, lon, lat); + HashMap params = new HashMap<>(); + params.put("hourlysteps", "24"); + CaiYunForecastResponse response = caiYunRequest(url, params, CaiYunForecastResponse.class); + return response.getHourlyResult().getHourly(); + } + + + public List getCaiYunWarning(double lat, double lon) throws Exception { + String url = String.format("https://api.caiyunapp.com/v2.6/%s/%.2f,%.2f/realtime", key, lon, lat); + HashMap params = new HashMap<>(); + params.put("alert", "true"); + CaiYunWarningResponse response = caiYunRequest(url, params, CaiYunWarningResponse.class); + return response.getResult().getAlert().getContent(); + } + + private NowWeatherStatus buildNowSurfaceWeatherStatus(CaiYunNowResponse nowResponse) { + NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); + try { + RealTime realtime = nowResponse.getNowResult().getRealtime(); + String time = DateTimeUtils.getTimeStringFromMilli(nowResponse.getServerTime()*1000, nowResponse.getTimeZone()); + nowWeatherStatus.setTime(time); + nowWeatherStatus.setWeatherText(CaiYunInfo.getSkyConName(realtime.getSkycon())); + nowWeatherStatus.setTemp(realtime.getTemperature()); + nowWeatherStatus.setWindSpeed((float) MeteoUtils.kmPerHour2mPerSecond(realtime.getWind().getSpeed())); + nowWeatherStatus.setWind360(realtime.getWind().getDirection()); + nowWeatherStatus.setHumidity(realtime.getHumidity()); + nowWeatherStatus.setCloud(realtime.getCloudrate()); + + CaiYunNowResponse.NowResult.Precipitation.Local localPrecipitation = realtime.getPrecipitation().getLocal(); + if ("ok".equals(localPrecipitation.getStatus())) { + nowWeatherStatus.setPrecip(localPrecipitation.getIntensity()); + } + return nowWeatherStatus; + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.CAI_YUN_DATA_PARSE_ERROR); + } + } + + private TimeSeriesDataset buildForecastSeries(Hourly hourly) { + try { + TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(24); + for (int i = 0; i < 24; i++) { + timeSeriesDataset.getTime().add(hourly.getTemperature().get(i).getDatetime()); + timeSeriesDataset.getTemp()[i] = hourly.getTemperature().get(i).getValue(); + timeSeriesDataset.getWindSpeed()[i] = (float) MeteoUtils.kmPerHour2mPerSecond(hourly.getWind().get(i).getSpeed()); + timeSeriesDataset.getWind360()[i] = hourly.getWind().get(i).getDirection(); + timeSeriesDataset.getHumidity()[i] = hourly.getHumidity().get(i).getValue(); + timeSeriesDataset.getCloud()[i] = hourly.getCloudrate().get(i).getValue(); + timeSeriesDataset.getPrecip()[i] = hourly.getPrecipitation().get(i).getValue(); + } + return timeSeriesDataset; + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.CAI_YUN_DATA_PARSE_ERROR, e.getMessage()); + } + } + + private List buildSurfaceWarning(List warnings) { + try { + List surfaceWeatherWarnings = new ArrayList<>(); + for (AlertContent warning : warnings) { + SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning(); + surfaceWeatherWarning.setPubTime(DateTimeUtils.getTimeStringFromMilli(warning.getPubtimestamp())); + surfaceWeatherWarning.setTitle(warning.getTitle()); + surfaceWeatherWarning.setText(warning.getDescription()); + surfaceWeatherWarning.setTypeName(CaiYunInfo.getAlertName(warning.getCode())); + surfaceWeatherWarning.setSeverityColor(CaiYunInfo.getAlertColor(warning.getCode())); + surfaceWeatherWarning.setSource(warning.getSource()); + surfaceWeatherWarnings.add(surfaceWeatherWarning); + } + return surfaceWeatherWarnings; + } catch (Exception e) { + throw new AppExcpetion(ErrorCode.CAI_YUN_DATA_PARSE_ERROR); + } + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java new file mode 100644 index 0000000..df44a76 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImpl.java @@ -0,0 +1,273 @@ +package com.htfp.weather.web.service.surfaceapi; + +import com.htfp.weather.utils.DateTimeUtils; +import com.htfp.weather.utils.MeteoUtils; +import com.htfp.weather.web.exception.AppExcpetion; +import com.htfp.weather.web.exception.ErrorCode; +import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import com.htfp.weather.web.pojo.response.NowWeatherStatus; +import com.htfp.weather.web.pojo.hefeng.*; +import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse.HeFengNow; +import com.htfp.weather.web.pojo.hefeng.HeFengForecastResponse.HeFengForecastHour; +import com.htfp.weather.web.pojo.hefeng.HeFengWarningResponse.HeFengWarning; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.util.*; + +import static com.htfp.weather.utils.HttpClientUtils.*; + +/** + * @Author : shiyi + * @Date : 2024/1/23 10:57 + * @Description : 和风天气服务 + */ +@Service("hefeng") +@Slf4j +public class HeFengServiceImpl implements ISurfaceDataService { + @Value("${hefeng.key}") + private String key; + private final HashMap variableNameMap = + new HashMap() {{ + put("temp", "temp"); + put("windSpeed", "windSpeed"); + put("windDir", "windDir"); + // put("windScale", "windScale"); + put("humidity", "humidity"); + put("cloud", "cloud"); + put("pressure", "pressure"); + put("precip", "precip"); + }}; + static final String STATUS_OK = "200"; + static final String STATUS_NO_DATA = "204"; + + public T heFengRequest(String url, HashMap params, Class responseClass) throws Exception { + // String signature = getSignature(params, key); + // HashMap params = new HashMap<>(); + // params.put("sign", signature); + params.put("key", key); + try { + T response = sendGet(url, params, responseClass); + if (response != null && STATUS_OK.equals(response.getCode())) { + return response; + } else if (response == null) { + log.error("[和风天气] 请求失败"); + throw new AppExcpetion(ErrorCode.HE_FENG_REQUEST_ERROR); + } else if (STATUS_NO_DATA.equals(response.getCode())) { + log.error("[和风天气] {}: {}", ErrorCode.HE_FENG_THIS_AREA_HAVE_NO_DATA.getMsg(), response); + throw new AppExcpetion(ErrorCode.HE_FENG_THIS_AREA_HAVE_NO_DATA); + } else { + log.error("[和风天气] {}: {}", ErrorCode.HE_FENG_THIS_AREA_HAVE_NO_DATA.getMsg(), response); + throw new AppExcpetion(ErrorCode.HE_FENG_THIS_AREA_HAVE_NO_DATA); + } + } catch (Exception e) { + log.error("[和风天气] 请求结果处理错误, {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + /** + * 获取地面指定位置的天气状况 + * + * @throws Exception + */ + @Override + public NowWeatherStatus getNowSurfaceWeatherStatus(double lat, double lon) throws Exception { + HeFengNow nowWeather = getFeFengNowWeather(lat, lon); + return buildNowSurfaceWeatherStatus(nowWeather); + } + + /** + * 获取地面指定位置某变量的24小时预报序列 + * + * @throws Exception + */ + @Override + public TimeSeriesDataset getForecastSeries(double lat, double lon) throws Exception { + List hourly = getHeFengForecast24(lat, lon); + return buildForecastSeries(hourly); + } + + @Override + public List getSurfaceWarning(double lat, double lon) throws Exception { + List heFengWarnings = getHeFengWarning(lat, lon); + return buildSurfaceWarning(heFengWarnings); + } + + public HeFengNow getFeFengNowWeather(double lat, double lon) throws Exception { + String url = "https://devapi.qweather.com/v7/grid-weather/now"; + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + + HeFengNowResponse response = heFengRequest(url, params, HeFengNowResponse.class); + return response.getNow(); + } + + public List getHeFengForecast24(double lat, double lon) throws Exception { + String url = "https://devapi.qweather.com/v7/grid-weather/24h"; + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + + HeFengForecastResponse response = heFengRequest(url, params, HeFengForecastResponse.class); + return response.getHourly(); + + } + + public List getHeFengWarning(double lat, double lon) throws Exception { + String url = "https://devapi.qweather.com/v7/warning/now"; + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + HeFengWarningResponse response = heFengRequest(url, params, HeFengWarningResponse.class); + return response.getWarning(); + } + + private TimeSeriesDataset buildForecastSeries(List hourly) { + TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(hourly.size()); + for (int i = 0; i < hourly.size(); i++) { + HeFengForecastHour heFengForecastHour = hourly.get(i); + processHeFengData(heFengForecastHour); + timeSeriesDataset.getTime().add(heFengForecastHour.getFxTime()); + timeSeriesDataset.getTemp()[i] = heFengForecastHour.getTemp(); + timeSeriesDataset.getWindSpeed()[i] = heFengForecastHour.getWindSpeed(); + timeSeriesDataset.getWind360()[i] = heFengForecastHour.getWind360(); + // timeSeriesDataset.getWindScale()[i] = heFengForecastHour.getWindScale(); + timeSeriesDataset.getHumidity()[i] = heFengForecastHour.getHumidity(); + // timeSeriesDataset.getPressure()[i] = heFengForecastHour.getPressure(); + timeSeriesDataset.getCloud()[i] = heFengForecastHour.getCloud(); + timeSeriesDataset.getPrecip()[i] = heFengForecastHour.getPrecip(); + } + return timeSeriesDataset; + } + + /** + * 从和风返回的数据中解析指定变量序列 + */ + // private TimeSeries buildForecastSeries(List hourly, String variableName) throws IllegalAccessException, NoSuchFieldException { + // TimeSeries timeSeries = new TimeSeries<>(); + // List time = new ArrayList<>(); + // List variable = new ArrayList<>(); + // String heFengName = variableNameMap.get(variableName); + // if (heFengName == null) { + // log.warn("[和风天气] 未找到{}对应的变量名", variableName); + // throw new AppExcpetion("[和风天气] 未找到" + variableName + "对应的变量名"); + // } + // Field field = BaseHeFengData.class.getDeclaredField(heFengName); + // field.setAccessible(true); + // for (HeFengForecastHour heFengForecastHour : hourly) { + // processHeFengData(heFengForecastHour); + // time.add(heFengForecastHour.getFxTime()); + // variable.add((Float) field.get(heFengForecastHour)); + // } + // timeSeries.setName(variableName); + // timeSeries.setTime(time); + // timeSeries.setData(variable); + // return timeSeries; + // } + private NowWeatherStatus buildNowSurfaceWeatherStatus(HeFengNow now) { + NowWeatherStatus nowWeatherStatus = new NowWeatherStatus(); + processHeFengData(now); + nowWeatherStatus.setTime(now.getObsTime()); + nowWeatherStatus.setWeatherText(now.getText()); + nowWeatherStatus.setTemp(now.getTemp()); + nowWeatherStatus.setWindSpeed(now.getWindSpeed()); + nowWeatherStatus.setWind360(now.getWind360()); + // nowSurfaceWeatherStatus.setWindDir(now.getWindDir()); + // nowSurfaceWeatherStatus.setWindScale(now.getWindScale()); + nowWeatherStatus.setHumidity(now.getHumidity()); + // nowWeatherStatus.setPressure(now.getPressure()); + nowWeatherStatus.setCloud(now.getCloud()); + nowWeatherStatus.setPrecip(now.getPrecip()); + return nowWeatherStatus; + } + + private List buildSurfaceWarning(List heFengWarnings) { + List surfaceWeatherWarnings = new ArrayList<>(); + for (HeFengWarning heFengWarning : heFengWarnings) { + SurfaceWeatherWarning surfaceWeatherWarning = new SurfaceWeatherWarning(); + surfaceWeatherWarning.setPubTime(heFengWarning.getPubTime()); + // surfaceWeatherWarning.setStartTime(heFengWarning.getStartTime()); + surfaceWeatherWarning.setTitle(heFengWarning.getTitle()); + surfaceWeatherWarning.setText(heFengWarning.getText()); + surfaceWeatherWarning.setTypeName(heFengWarning.getTypeName()); + surfaceWeatherWarning.setSeverityColor(heFengWarning.getSeverityColor()); + surfaceWeatherWarning.setSource(heFengWarning.getSender()); + surfaceWeatherWarnings.add(surfaceWeatherWarning); + } + return surfaceWeatherWarnings; + } + + /** + * 从和风返回的数据中对时间进行处理，风速单位处理 + * + * @param data + * @return + */ + private BaseHeFengData processHeFengData(BaseHeFengData data) { + String format = "yyyy-MM-dd'T'HH:mmxxx"; + if (data instanceof HeFengForecastHour) { + String fxTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengForecastHour) data).getFxTime(), format); + ((HeFengForecastHour) data).setFxTime(fxTime); + } else if (data instanceof HeFengNow) { + String obsTime = DateTimeUtils.getBJTDateTimeStringFromUTC(((HeFengNow) data).getObsTime(), format); + ((HeFengNow) data).setObsTime(obsTime); + } + float windSpeed = (float) MeteoUtils.kmPerHour2mPerSecond(data.getWindSpeed()); + data.setWindSpeed(windSpeed); + return data; + } + + /** + * 和风天气签名生成算法-JAVA版本 + * + * @param params 请求参数集，所有参数必须已转换为字符串类型 + * @param secret 签名密钥（用户的认证key） + * @return 签名 + * @throws IOException + */ + public String getSignature(HashMap params, String secret) throws Exception { + // 先将参数以其参数名的字典序升序进行排序 + Map sortedParams = new TreeMap<>(params); + Set> entrys = sortedParams.entrySet(); + + // 遍历排序后的字典，将所有参数按"key=value"格式拼接在一起 + StringBuilder baseString = new StringBuilder(); + for (Map.Entry param : entrys) { + // sign参数 和 空值参数 不加入算法 + if (param.getValue() != null && !"".equals(param.getKey().trim()) && !"sign".equals(param.getKey().trim()) && !"key".equals(param.getKey().trim()) && !"".equals(param.getValue().trim())) { + baseString.append(param.getKey().trim()).append("=").append(param.getValue().trim()).append("&"); + } + } + if (baseString.length() > 0) { + baseString.deleteCharAt(baseString.length() - 1).append(secret); + } + // 使用MD5对待签名串求签 + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + byte[] bytes = md5.digest(baseString.toString().getBytes(StandardCharsets.UTF_8)); + return new String(encodeHex(bytes)); + } catch (GeneralSecurityException ex) { + throw ex; + } + } + + public char[] encodeHex(byte[] data) { + int l = data.length; + char[] out = new char[l << 1]; + int i = 0; + char[] toDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + for (int var5 = 0; i < l; ++i) { + out[var5++] = toDigits[(240 & data[i]) >>> 4]; + out[var5++] = toDigits[15 & data[i]]; + } + return out; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/ISurfaceDataService.java b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/ISurfaceDataService.java new file mode 100644 index 0000000..7eb88d8 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/service/surfaceapi/ISurfaceDataService.java @@ -0,0 +1,20 @@ +package com.htfp.weather.web.service.surfaceapi; + +import com.htfp.weather.web.pojo.response.NowWeatherStatus; +import com.htfp.weather.web.pojo.response.SurfaceWeatherWarning; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; + +import java.util.List; + +/** + * @Author : shiyi + * @Date : 2024/4/30 10:02 + * @Description : + */ +public interface ISurfaceDataService { + NowWeatherStatus getNowSurfaceWeatherStatus(double lat, double lon) throws Exception; + + TimeSeriesDataset getForecastSeries(double lat, double lon) throws Exception; + + List getSurfaceWarning(double lat, double lon) throws Exception; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeStr.java b/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeStr.java new file mode 100644 index 0000000..d419042 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeStr.java @@ -0,0 +1,28 @@ +package com.htfp.weather.web.valid; + + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.*; + +/** + * @Author : shiyi + * @Date : 2024/5/10 17:39 + * @Description : + */ + +@Target({ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Constraint(validatedBy = DateTimeValidator.class) +public @interface DateTimeStr { + + String message() default "{javax.validation.constraints.DateTimeStr.message}"; + + + String format() default "yyyy-MM-dd HH:mm:ss"; + + Class[] groups() default {}; + + Class[] payload() default {}; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeValidator.java b/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeValidator.java new file mode 100644 index 0000000..eace7c8 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/valid/DateTimeValidator.java @@ -0,0 +1,45 @@ +package com.htfp.weather.web.valid; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import java.lang.annotation.Annotation; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; + +/** + * @Author : shiyi + * @Date : 2024/5/10 17:38 + * @Description : + */ + + +public class DateTimeValidator implements ConstraintValidator{ + + private DateTimeStr dateTimeStr; + + + @Override + public void initialize(DateTimeStr dateTimeStr) { + this.dateTimeStr=dateTimeStr; + } + + @Override + public boolean isValid(String value, ConstraintValidatorContext context) { + // 如果 value 为空则不进行格式验证，为空验证可以使用 @NotBlank @NotNull @NotEmpty 等注解来进行控制，职责分离 + if (value == null) { + return true; + } + String format = dateTimeStr.format(); + + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format); + + try { + dateTimeFormatter.parse(value); + } catch (Exception e){ + return false; + } + + + return true; + } +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValid.java b/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValid.java new file mode 100644 index 0000000..6768f83 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValid.java @@ -0,0 +1,27 @@ +package com.htfp.weather.web.valid; + +/** + * @Author : shiyi + * @Date : 2024/4/28 16:22 + * @Description : + */ + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.*; + +@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.ANNOTATION_TYPE }) +@Retention(RetentionPolicy.RUNTIME) +@Constraint(validatedBy = {EnumValidator.class}) +@Documented +public @interface EnumValid { + String message() default ""; + + Class[] groups() default {}; + + Class[] payload() default {}; + + Class[] enumClass() default {}; + + String usedField() default "getCode"; +} diff --git a/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValidator.java b/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValidator.java new file mode 100644 index 0000000..6f29e27 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/web/valid/EnumValidator.java @@ -0,0 +1,63 @@ +package com.htfp.weather.web.valid; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * @Author : shiyi + * @Date : 2024/4/28 16:23 + * @Description : + */ + +/** + * 校验入参是否为指定enum的值的实现方法 + */ +public class EnumValidator implements ConstraintValidator { + + String validField; + Class[] cls; // 枚举类 + + @Override + public void initialize(EnumValid constraintAnnotation) { + cls = constraintAnnotation.enumClass(); + validField = constraintAnnotation.usedField(); + } + + @Override + public boolean isValid(Object value, ConstraintValidatorContext context) { + if (value != null && !value.toString().isEmpty() && cls.length > 0) { + for (Class cl : cls) { + try { + if (cl.isEnum()) { + // 枚举类验证 + Object[] objs = cl.getEnumConstants(); + Method method = cl.getMethod("name"); + for (Object obj : objs) { + Object code = method.invoke(obj, null); + if (value.equals(code.toString())) { + return true; + } + } + Method codeMethod = cl.getMethod(validField); + for (Object obj : objs) { + Object code = codeMethod.invoke(obj, null); + if (value.toString().equals(code.toString())) { + return true; + } + } + } + } catch (NoSuchMethodException | IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + } + } else { + return true; + } + return false; + } + +} diff --git a/weather-service/src/main/resources/application-dev.yml b/weather-service/src/main/resources/application-dev.yml new file mode 100644 index 0000000..cf221bc --- /dev/null +++ b/weather-service/src/main/resources/application-dev.yml @@ -0,0 +1,6 @@ +server: + port: 8080 + +spring: + profiles: + include: weather diff --git a/weather-service/src/main/resources/application-weather.yml b/weather-service/src/main/resources/application-weather.yml new file mode 100644 index 0000000..d73b258 --- /dev/null +++ b/weather-service/src/main/resources/application-weather.yml @@ -0,0 +1,12 @@ +#和风天气 +hefeng: + key: 4df66bb82da04b5bb85624874209fc73 +caiyun: + key: Tc9tgOYr5jlPPlEw + +# tablestore +tablestore: + endpoint: https://gfs-test.cn-hangzhou.ots.aliyuncs.com + accessId: LTAI5tDphvXLfwKJEmVQqrVz + accessKey: ksioVP1S36PI6plkIIRN4A2xLB94uc + instanceName: gfs-test \ No newline at end of file diff --git a/weather-service/src/main/resources/application.yml b/weather-service/src/main/resources/application.yml new file mode 100644 index 0000000..cd4c264 --- /dev/null +++ b/weather-service/src/main/resources/application.yml @@ -0,0 +1,12 @@ +server: + port: 10323 +spring: + profiles: + include: weather + mvc: + servlet: + load-on-startup: 1 # 关闭 dispatcherServlet懒加载 + + thymeleaf: + cache: false + encoding: UTF-8 \ No newline at end of file diff --git a/weather-service/src/main/resources/config/gfsDataConfig.json b/weather-service/src/main/resources/config/gfsDataConfig.json new file mode 100644 index 0000000..999651a --- /dev/null +++ b/weather-service/src/main/resources/config/gfsDataConfig.json @@ -0,0 +1,11 @@ +{ + "duration" : 10, + "minLon" : 70.0, + "maxLon" : 140.0, + "minLat" : 0.0, + "maxLat" : 55.0, + "resolution" : 0.25, + "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ], + "pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000], + "saveRoot" : "GFSData" +} \ No newline at end of file diff --git a/weather-service/src/main/resources/config/tableConf.json b/weather-service/src/main/resources/config/tableConf.json new file mode 100644 index 0000000..2bb5d5b --- /dev/null +++ b/weather-service/src/main/resources/config/tableConf.json @@ -0,0 +1,12 @@ +{ + "dataTableName" : "gfs_data_table", + "metaTableName" : "gfs_meta_table", + "dataIndexName" : "gfs_data_index", + "metaIndexName" : "gfs_meta_table_index", + "dataDir" : "D:/HTFP/weather/GFSData/", + "variableList" : [ "temp", "cloud", "wind360", "windSpeed"], + "lonSize" : 281, + "latSize" : 221, + "levSize" : 10, + "timeToLive" : 2 +} \ No newline at end of file diff --git a/weather-service/src/main/resources/logback.xml b/weather-service/src/main/resources/logback.xml new file mode 100644 index 0000000..9f9102b --- /dev/null +++ b/weather-service/src/main/resources/logback.xml @@ -0,0 +1,45 @@ + + + + + + + %highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n + + + + debug + + + + + + + + ./log/%d{yyyy-MM-dd}.%i.log + + 30 + 10MB + + + + %highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n + %-5level %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{50} - %msg%n + UTF-8 + + + + + + + + + + + + + + + + + diff --git a/weather-service/src/main/resources/static/index.html b/weather-service/src/main/resources/static/index.html new file mode 100644 index 0000000..89bb8ba --- /dev/null +++ b/weather-service/src/main/resources/static/index.html @@ -0,0 +1,6 @@ + + +

hello word!!!

+

this is a html page

+ + \ No newline at end of file diff --git a/weather-service/src/main/resources/templates/index.html b/weather-service/src/main/resources/templates/index.html new file mode 100644 index 0000000..f5c6d26 --- /dev/null +++ b/weather-service/src/main/resources/templates/index.html @@ -0,0 +1,60 @@ + + + + + Title + + + + +
+

Tablestore 表格存储配置信息

+

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
数据表名称 dataTableName
属性表名称 metaTableName
数据索引名称 dataIndexName
属性索引名称 metaIndexName
数据存储路径根目录 dataDir edit
纬度网格数 latSize
经度网格数 lonSize
高度层数 levSize
+
+ + + + + + \ No newline at end of file diff --git a/weather-service/src/main/resources/templates/wrong.html b/weather-service/src/main/resources/templates/wrong.html new file mode 100644 index 0000000..b6c1dec --- /dev/null +++ b/weather-service/src/main/resources/templates/wrong.html @@ -0,0 +1,10 @@ + + + + + Title + + +请求错误 + + \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/WeatherServiceApplicationTests.java b/weather-service/src/test/java/com/htfp/weather/WeatherServiceApplicationTests.java new file mode 100644 index 0000000..75a2d3b --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/WeatherServiceApplicationTests.java @@ -0,0 +1,20 @@ +package com.htfp.weather; + +import com.htfp.weather.griddata.operation.CreateTable; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest @Slf4j +class WeatherServiceApplicationTests { + + @Test + void contextLoads() { + } + + @Test + void run() throws Exception { + CreateTable example = new CreateTable(); + System.out.println(example); + } +} diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java new file mode 100644 index 0000000..41957c7 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDataConfigTest.java @@ -0,0 +1,24 @@ +package com.htfp.weather.download; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/4/21 16:28 + * @Description : + */ +class GfsDataConfigTest { + + @Test + void writeConfig() throws IOException { + GfsDataConfig gfsDataConfig = new GfsDataConfig(); + gfsDataConfig.readConfig(); + gfsDataConfig.setDuration(100); + System.out.println(gfsDataConfig); + gfsDataConfig.writeConfig(gfsDataConfig,"test.json"); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java new file mode 100644 index 0000000..5e38a89 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/download/GfsDownloaderTest.java @@ -0,0 +1,54 @@ +package com.htfp.weather.download; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/4/25 16:15 + * @Description : + */ +class GfsDownloaderTest { + + public static GfsDataConfig gfsDataConfig = new GfsDataConfig(); + static { + gfsDataConfig.readConfig(); + } + GfsDownloader gfsDownloader = new GfsDownloader(gfsDataConfig); + + + @Test + void iniTimeSetting() { + gfsDownloader.iniTimeSetting(); + } + + @Test + void download() { + gfsDownloader.iniTimeSetting(); + List fileInfoList = gfsDownloader.getFilesInfo(); + gfsDownloader.download(fileInfoList.get(0)); + } + + @Test + void downloadAll() { + gfsDownloader.iniTimeSetting(); + List fileInfoList = gfsDownloader.getFilesInfo(); + boolean isComplete = gfsDownloader.downloadAll(fileInfoList); + if (isComplete) { + System.out.println("下载完成"); + } + } + + @Test + void getFilesInfo() { + gfsDownloader.iniTimeSetting(); + List fileInfoList = gfsDownloader.getFilesInfo(); + for (FileInfo fileInfo :fileInfoList) { + System.out.println(fileInfo); + } + } + +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java new file mode 100644 index 0000000..a3f3055 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataFetcherTest.java @@ -0,0 +1,42 @@ +package com.htfp.weather.griddata.operation; + +import com.htfp.weather.griddata.common.GfsVariableNameEnum; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +/** + * @Author : shiyi + * @Date : 2024/2/2 16:01 + * @Description : + */ +@Slf4j @SpringBootTest +class GfsDataFetcherTest { + String dataSetId = "UTC-20240508.00"; + @Resource + GfsDataFetcher gfsDataFetcher; + + @Test + void queryByTableStore() { + } + + @Test + void getPlane() throws Exception { + log.info("获取平面"); + gfsDataFetcher.getPlane(dataSetId, GfsVariableNameEnum.getGfsVariableName("Cloud"),0, 0, 10, 10, 10, 10); + } + + @Test + void getProfile() throws Exception { + log.info("获取廓线"); + gfsDataFetcher.getProfile(dataSetId, GfsVariableNameEnum.getGfsVariableName("Temp"), 9,0, 0); + } + + // @Test + // void getSeries() throws Exception { + // log.info("获取时间序列"); + // gfsDataFetcher.getSeries(dataSetId, GfsVariableNameEnum.getGfsVariableName("Temp"), 0, 10, 10); + // } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataImportTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataImportTest.java new file mode 100644 index 0000000..05072b4 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/griddata/operation/GfsDataImportTest.java @@ -0,0 +1,52 @@ +package com.htfp.weather.griddata.operation; + +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import ucar.nc2.Attribute; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import javax.annotation.Resource; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import static java.time.format.DateTimeFormatter.ISO_DATE_TIME; +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/3/1 18:27 + * @Description : + */ +@SpringBootTest +class GfsDataImportTest { + @Resource + GfsDataImport gfsDataImport; + @Test + void getRefTime() throws IOException { + NetcdfFile ncFile = NetcdfFiles.open("C:/Users/shi_y/Desktop/GFSData/UTC-20230905/BJT-20230905-0900.grib2"); + Variable time = ncFile.findVariable("reftime"); + if ( time!= null) { + int hour = (int) time.read().getDouble(0); + Attribute units = time.findAttribute("units"); + if (units != null) { + String stringValue = units.getStringValue(); + if (StringUtils.isNotBlank(stringValue)) { + ZonedDateTime zonedDateTime = ZonedDateTime.parse(stringValue.substring(11)).plusHours(hour); + zonedDateTime.plusHours(hour); + } + } + } + } + + @Test + void importData() throws Exception { + OffsetDateTime refTime = OffsetDateTime.parse("2024-05-08T00:00+00:00"); + gfsDataImport.importData(refTime); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/operation/QueryMetaTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/operation/QueryMetaTest.java new file mode 100644 index 0000000..7541efe --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/griddata/operation/QueryMetaTest.java @@ -0,0 +1,30 @@ +package com.htfp.weather.griddata.operation; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +/** + * @Author : shiyi + * @Date : 2024/5/6 18:41 + * @Description : + */ +@SpringBootTest +class QueryMetaTest { + @Resource + QueryMeta queryMeta; + @Resource + GfsDataFetcher gfsDataFetcher; + @Resource + GfsDataImport gfsDataImport; + @Test + void queryRefTime() throws Exception { + try { + System.out.println("Query..."); + queryMeta.queryLastReferenceTime(); + } finally { + queryMeta.close(); + } + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java new file mode 100644 index 0000000..34540a2 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/griddata/utils/GfsUtilsTest.java @@ -0,0 +1,27 @@ +package com.htfp.weather.griddata.utils; + +import com.htfp.weather.utils.DateTimeUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/3/4 10:21 + * @Description : + */ +class GfsUtilsTest { + + @Test + void getRefTime() throws IOException { + NetcdfFile ncFile = NetcdfFiles.open("C:/Users/shi_y/Desktop/GFSData/UTC-20230905/BJT-20230905-0900.grib2"); + String refTime = GfsUtils.getRefTime(ncFile); + assertEquals("2023-09-05T00:00+00:00", refTime); + Assertions.assertEquals("2023-09-05T08:00+08:00", DateTimeUtils.getBJTDateTimeStringFromUTC(refTime, CoordinateUtils.DATE_TIME_PATTERN)); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java new file mode 100644 index 0000000..6cb8c50 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcesserTest.java @@ -0,0 +1,23 @@ +package com.htfp.weather.schedule; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/5/17 11:44 + * @Description : + */ +@SpringBootTest +class GridDataProcesserTest { + @Resource + GridDataProcesser gridDataProcesser; + @Test + void dailyDataProcess() { + gridDataProcesser.clearExpiredData(); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/utils/DateTimeUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/utils/DateTimeUtilsTest.java new file mode 100644 index 0000000..6c84fd1 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/utils/DateTimeUtilsTest.java @@ -0,0 +1,49 @@ +package com.htfp.weather.utils; + + +import org.junit.jupiter.api.Test; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +/** + * @Author : shiyi + * @Date : 2024/1/24 9:07 + * @Description : + */ +class DateTimeUtilsTest { + + @Test + void convertZonedDateTime() { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"); + + String timeZoneString = "2024-01-23T11:00:00+00:00"; + // 使用 OffsetDateTime 的 parse() 方法，将时区字符串转换为 OffsetDateTime 对象，表示东八区时区的 2024-01-23T11:00+08:00 + OffsetDateTime offsetDateTime = OffsetDateTime.parse(timeZoneString, formatter); + + // 使用 OffsetDateTime 的 withOffsetSameInstant() 方法，将东八区时区转换为0时区 + OffsetDateTime utc = offsetDateTime.withOffsetSameInstant(ZoneOffset.ofHours(8)); + + // 使用 OffsetDateTime 的 format() 方法，将 OffsetDateTime 对象转换为字符串 + String utcStr = utc.format(formatter); + + // 打印结果 + System.out.println("input: " + timeZoneString); + System.out.println("output: " + utc); + } + + @Test + void getBJTDateTimeFromUTC() { + String timeZoneString = "2024-01-23T11:00"; + System.out.println(DateTimeUtils.getBJTDateTimeStringFromUTC(timeZoneString, "yyyy-MM-dd'T'HH:mm")); + } + + @Test + void getTimestampMillli() { + long utc20230701 = DateTimeUtils.getTimestampMilli("UTC20230701", "'UTC-'yyyyMMdd"); + System.out.println(utc20230701); + } + + +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/utils/HttpClientUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/utils/HttpClientUtilsTest.java new file mode 100644 index 0000000..ed740a7 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/utils/HttpClientUtilsTest.java @@ -0,0 +1,93 @@ +package com.htfp.weather.utils; + +import com.htfp.weather.web.pojo.hefeng.HeFengForecastResponse; +import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; + +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/** + * @Author : shiyi + * @Date : 2024/1/23 17:11 + * @Description : + */ +@Slf4j +class HttpClientUtilsTest { + String key = "4df66bb82da04b5bb85624874209fc73"; + double lon = 116.4074; + double lat = 39.9042; + + @Test + public void sendGetTest1() throws Exception { + String url = "https://devapi.qweather.com/v7/grid-weather/now"; + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + params.put("key", key); + HeFengNowResponse heFengNowResponse = HttpClientUtils.sendGet(url, params, HeFengNowResponse.class); + if (heFengNowResponse == null || !heFengNowResponse.getCode().startsWith("2")){ + log.error("请求失败"); + return; + } + System.out.println(heFengNowResponse.getNow()); + } + + @Test + public void sendGetTest2() throws Exception { + // 权限错误 + String url = "https://devapi.qweather.com/v7/grid-weather/24h"; + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + params.put("key", key); + HeFengForecastResponse response = HttpClientUtils.sendGet(url, params, HeFengForecastResponse.class); + if (response == null || !response.getCode().startsWith("2")){ + log.error("请求失败"); + return; + } + System.out.println(response.getHourly()); + } + + @Test + public void sendGetTest3() throws IOException { + // 参数错误 + String format = "https://devapi.qweather.com/v7/grid-weather/now"; + String url = String.format(format, lon, lat, key); + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + params.put("key", key); + System.out.println(HttpClientUtils.sendGet(url, params)); + } + + @Test + public void sendGetTest4() throws Exception { + // 参数错误 + String format = "https://devapi.qweather.com/v7/grid-weather/now"; + String url = String.format(format, lon, lat, key); + String location = String.format("%.2f,%.2f", lon, lat); + HashMap params = new HashMap<>(); + params.put("location", location); + params.put("key", key); + String jsonStr = HttpClientUtils.sendGet(url, params); + System.out.println(JSONUtils.json2pojo(jsonStr, HeFengNowResponse.class)); + // assertThrows(IOException.class, ()->{HttpClientUtils.sendGet(url, params);}); + } + + @Test + public void sendGetTest5() throws Exception { + // 参数错误 + String url = "http://localhost:10323/htfp/weather/index"; + HashMap params = new HashMap<>(); + params.put("lat", "33.3"); + params.put("lon", "127.1"); + System.out.println(HttpClientUtils.sendGet(url, params, HeFengNowResponse.class)); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/utils/MeteoUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/utils/MeteoUtilsTest.java new file mode 100644 index 0000000..7f07850 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/utils/MeteoUtilsTest.java @@ -0,0 +1,57 @@ +package com.htfp.weather.utils; + +import org.junit.jupiter.api.Test; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/5/8 18:12 + * @Description : + */ +class MeteoUtilsTest { + + @Test + void calculateWindSpeed() { + } + + @Test + void calculateWindDirection() { + float u = 10.0F; + float v = 10.0F; + System.out.println(MeteoUtils.calculateWindDirection(u, v)); + } + + @Test + void testCalculateWindSpeed() throws IOException { + try (NetcdfFile ncFile = NetcdfFiles.open("D:\\HTFP\\weather\\GFSData\\UTC-20240512.18\\BJT-20240513.09-from-UTC-20240512.18+7.grib2")) { + + Array uwnd = ncFile.findVariable("u-component_of_wind_isobaric").read(new int[]{0, 0, 0, 0}, new int[]{1, 10, 221, 281}); + Array vwnd = ncFile.findVariable("v-component_of_wind_isobaric").read(new int[]{0, 0, 0, 0}, new int[]{1, 10, 221, 281}); + // long l1 = measureExecutionTime(() -> MeteoUtils.calculateWindSpeed1(uwnd, vwnd)); + long l2 = measureExecutionTime(() -> MeteoUtils.calculateWindSpeed(uwnd, vwnd)); + // System.out.println("java循环计算平均单次耗时：" + l1 + "ms"); + System.out.println("djl计算平均单次耗时：" + l2 + "ms"); + } catch (InvalidRangeException e) { + throw new RuntimeException(e); + } + } + + public static long measureExecutionTime(Runnable task) { + long startTime = System.currentTimeMillis(); + + // 执行任务 + for (int i = 0; i < 10; i++) { + task.run(); + } + long endTime = System.currentTimeMillis(); + return (long) ((endTime - startTime)/10.); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/utils/NdArrayUtilsTest.java b/weather-service/src/test/java/com/htfp/weather/utils/NdArrayUtilsTest.java new file mode 100644 index 0000000..2ba0167 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/utils/NdArrayUtilsTest.java @@ -0,0 +1,82 @@ +package com.htfp.weather.utils; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.htfp.weather.utils.NdArrayUtils.findNearestIndex; +import static org.junit.jupiter.api.Assertions.*; +/** +* @Author : shiyi +* @Date : 2024/5/10 11:49 +* @Description : +*/class NdArrayUtilsTest { + + @Test + public void testEmptyList() { + double[] list = new double[] {}; + double target = 3.5; + int expected = -1; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testSingleElementList() { + double[] list = new double[] {1.0}; + double target = 3.5; + int expected = 0; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testThreeElementList() { + double[] list = new double[] {1.0, 2.0, 3.0}; + double target = 2.4; + int expected = 1; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testFiveElementList() { + double[] list = new double[] {1.0, 2.0, 3.0, 4.0, 5.0}; + double target = 3.5; + int expected = 2; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testTenElementList() { + double[] list = new double[] {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + double target = 5.5; + int expected = 4; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testStepElementList() { + double[] list = new double[] {1.0, 1.0, 2.0, 2.0, 3.0, 4.0, 4.0, 5.0, 6.0, 7.0}; + double target = 1.7; + int expected = 2; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + } + + @Test + public void testLongElementList() { + double[] list = new double[1000] ; + for (int i = 0; i < list.length; i++) { + list[i] = 0 + i*0.025; + } + double target = 18.524; + int expected = 741; + int result = findNearestIndex(list, target); + assertEquals(expected, result); + System.out.println("result:" + list[expected]); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/web/controller/UpperWeatherControllerTest.java b/weather-service/src/test/java/com/htfp/weather/web/controller/UpperWeatherControllerTest.java new file mode 100644 index 0000000..3f60702 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/web/controller/UpperWeatherControllerTest.java @@ -0,0 +1,17 @@ +package com.htfp.weather.web.controller; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/5/9 18:22 + * @Description : + */ +class UpperWeatherControllerTest { + + @Test + void queryProfile() { + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImplTest.java b/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImplTest.java new file mode 100644 index 0000000..6af8f13 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/CaiYunServiceImplTest.java @@ -0,0 +1,37 @@ +package com.htfp.weather.web.service.surfaceapi; + +import org.junit.jupiter.api.Test; + +/** + * @Author : shiyi + * @Date : 2024/4/15 19:48 + * @Description : + */ +// @SpringBootTest +class CaiYunServiceImplTest { + + private final CaiYunServiceImpl caiYunServiceImpl = new CaiYunServiceImpl(); + + + @Test + void getNowSurfaceWeatherStatus() throws Exception { + double lon = 116.4074; + double lat = 39.9042; + System.out.println(caiYunServiceImpl.getNowSurfaceWeatherStatus(lat, lon)); + } + + + @Test + void getForecastSeries() throws Exception { + double lon = 116.4074; + double lat = 39.9042; + System.out.println(caiYunServiceImpl.getForecastSeries(lat, lon)); + } + + @Test + void getSurfaceWarning() throws Exception { + double lon = 101.6656; + double lat = 39.2072; + System.out.println(caiYunServiceImpl.getSurfaceWarning(lat, lon)); + } +} \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImplTest.java b/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImplTest.java new file mode 100644 index 0000000..1ffc504 --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/web/service/surfaceapi/HeFengServiceImplTest.java @@ -0,0 +1,50 @@ +package com.htfp.weather.web.service.surfaceapi; + +import com.htfp.weather.web.pojo.hefeng.HeFengNowResponse; +import com.htfp.weather.web.pojo.response.NowWeatherStatus; +import com.htfp.weather.web.pojo.response.TimeSeriesDataset; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +/** + * @Author : shiyi + * @Date : 2024/1/23 11:07 + * @Description : + */ +@SpringBootTest +@Slf4j +class HeFengServiceImplTest { + @Resource + private HeFengServiceImpl heFengServiceImpl; + @Value("${hefeng.key}") + private String key; + private final String urlFormat = "https://devapi.qweather.com/v7/grid-weather/now?location=%.2f,%.2f&key=%s"; + + @Test + public void getCurrentSurfaceWeatherStatus() throws Exception { + double lon = 116.4074; + double lat = 39.9042; + NowWeatherStatus nowWeather = heFengServiceImpl.getNowSurfaceWeatherStatus(lat, lon); + System.out.println(nowWeather); + } + + @Test + void getFourcasetSeries() throws Exception { + double lon = 116.4074; + double lat = 39.9042; + TimeSeriesDataset forecastSeries = heFengServiceImpl.getForecastSeries(lat, lon); + System.out.println(forecastSeries); + } + + @Test + void getNowWeather() throws Exception { + double lon = 116.4074; + double lat = 39.9042; + HeFengNowResponse.HeFengNow nowWeather = heFengServiceImpl.getFeFengNowWeather(lat, lon); + System.out.println(nowWeather); + } +} \ No newline at end of file diff --git a/weather-service/src/test/resources/application-dev.yml b/weather-service/src/test/resources/application-dev.yml new file mode 100644 index 0000000..cf221bc --- /dev/null +++ b/weather-service/src/test/resources/application-dev.yml @@ -0,0 +1,6 @@ +server: + port: 8080 + +spring: + profiles: + include: weather diff --git a/weather-service/src/test/resources/application-weather.yml b/weather-service/src/test/resources/application-weather.yml new file mode 100644 index 0000000..d73b258 --- /dev/null +++ b/weather-service/src/test/resources/application-weather.yml @@ -0,0 +1,12 @@ +#和风天气 +hefeng: + key: 4df66bb82da04b5bb85624874209fc73 +caiyun: + key: Tc9tgOYr5jlPPlEw + +# tablestore +tablestore: + endpoint: https://gfs-test.cn-hangzhou.ots.aliyuncs.com + accessId: LTAI5tDphvXLfwKJEmVQqrVz + accessKey: ksioVP1S36PI6plkIIRN4A2xLB94uc + instanceName: gfs-test \ No newline at end of file diff --git a/weather-service/src/test/resources/application.yml b/weather-service/src/test/resources/application.yml new file mode 100644 index 0000000..cd4c264 --- /dev/null +++ b/weather-service/src/test/resources/application.yml @@ -0,0 +1,12 @@ +server: + port: 10323 +spring: + profiles: + include: weather + mvc: + servlet: + load-on-startup: 1 # 关闭 dispatcherServlet懒加载 + + thymeleaf: + cache: false + encoding: UTF-8 \ No newline at end of file diff --git a/weather-service/src/test/resources/config/gfsDataConfig.json b/weather-service/src/test/resources/config/gfsDataConfig.json new file mode 100644 index 0000000..999651a --- /dev/null +++ b/weather-service/src/test/resources/config/gfsDataConfig.json @@ -0,0 +1,11 @@ +{ + "duration" : 10, + "minLon" : 70.0, + "maxLon" : 140.0, + "minLat" : 0.0, + "maxLat" : 55.0, + "resolution" : 0.25, + "variables" : [ "DZDT", "RH", "TMP", "UGRD", "VGRD", "TCDC" ], + "pressureLevels" : [400, 500, 600, 700, 750, 800, 850, 925, 975, 1000], + "saveRoot" : "GFSData" +} \ No newline at end of file diff --git a/weather-service/src/test/resources/config/tableConf.json b/weather-service/src/test/resources/config/tableConf.json new file mode 100644 index 0000000..b07253b --- /dev/null +++ b/weather-service/src/test/resources/config/tableConf.json @@ -0,0 +1,12 @@ +{ + "dataTableName" : "gfs_data_table", + "metaTableName" : "gfs_meta_table", + "dataIndexName" : "gfs_data_index", + "metaIndexName" : "gfs_meta_table_index", + "dataDir" : "D:/HTFP/weather/GFSData/", + "variableList" : [ "Temp", "Cloud", "WindSpeed", "WindDirection"], + "lonSize" : 281, + "latSize" : 221, + "levSize" : 10, + "timeToLive" : 2 +} \ No newline at end of file diff --git a/weather-service/src/test/resources/logback-test.xml b/weather-service/src/test/resources/logback-test.xml new file mode 100644 index 0000000..7dd680e --- /dev/null +++ b/weather-service/src/test/resources/logback-test.xml @@ -0,0 +1,47 @@ + + + + + + %highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n + + + + debug + + + + + + + + ./log/%d{yyyy-MM-dd}.%i.log + + 30 + 10MB + + + + %highlight(%-5level) %d{HH:mm:ss.SSS} %magenta([%thread]) %cyan(%logger{36}) - %msg%n + %-5level %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{50} - %msg%n + UTF-8 + + + + + + + + + + + + + + + + + + + + diff --git a/weather-service/src/test/resources/static/index.html b/weather-service/src/test/resources/static/index.html new file mode 100644 index 0000000..89bb8ba --- /dev/null +++ b/weather-service/src/test/resources/static/index.html @@ -0,0 +1,6 @@ + + +

hello word!!!

+

this is a html page

+ + \ No newline at end of file