后台管理新增一键下载并导入的接口

refactor
shiyi 9 months ago
parent 443a226fd7
commit 17a35e9ab2

@ -25,22 +25,22 @@ public abstract class BaseDataDownloader {
/**
*
*/
public abstract List<FileInfo> getFilesInfo();
public abstract List<DownLoadFileInfo> getFilesInfo();
/**
*
* @param fileInfo:
* @param downLoadFileInfo:
* @return /
* @throws IOException
*/
public abstract FileInfo download(FileInfo fileInfo) throws IOException;
public abstract DownLoadFileInfo download(DownLoadFileInfo downLoadFileInfo) throws IOException;
/**
*
* @param fileInfoList
* @param downLoadFileInfoList
* @return
*/
public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
public abstract List<DownLoadFileInfo> downloadAll(List<DownLoadFileInfo> downLoadFileInfoList);
}

@ -9,7 +9,7 @@ import lombok.ToString;
@Data
@ToString
public class FileInfo {
public class DownLoadFileInfo {
/**保存文件名*/
private String fileName;
/**数据起报时间*/
@ -26,7 +26,7 @@ public class FileInfo {
private String savePath;
private boolean isDownloadSuccess;
public FileInfo() {
public DownLoadFileInfo() {
}
}

@ -1,7 +1,7 @@
package com.htfp.weather.download.gfs;
import com.htfp.weather.download.BaseDataDownloader;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.DownLoadFileInfo;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.HttpClientUtils;
@ -26,7 +26,7 @@ import java.util.concurrent.Future;
/**
* @author shiyi
* @
* @Description GFS,
*/
@Slf4j
@Component("gfsDownloader")
@ -62,33 +62,33 @@ public class GfsDownloader extends BaseDataDownloader {
}
@Override
public FileInfo download(FileInfo fileInfo) {
return download0(fileInfo, 2);
public DownLoadFileInfo download(DownLoadFileInfo downLoadFileInfo) {
return download0(downLoadFileInfo, 2);
}
private FileInfo download0(FileInfo fileInfo, int retryNum) {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
File fileOut = new File(destDir, fileInfo.getFileName());
private DownLoadFileInfo download0(DownLoadFileInfo downLoadFileInfo, int retryNum) {
String url = downLoadFileInfo.getUrl();
File destDir = new File(downLoadFileInfo.getSavePath());
File fileOut = new File(destDir, downLoadFileInfo.getFileName());
log.info("[GFS Download] 文件下载中,保存至 {}", fileOut);
try {
// DONE: 改用okhttp FIXME 2024/6/8: 如果连接是httpsjar包启动下载会报错 javax.net.ssl.SSLException: Received fatal alert: internal_error
HttpClientUtils.downloadFileByUrl(url, fileOut.getPath());
log.info("[GFS Download] 文件下载成功: {}", fileOut);
fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
downLoadFileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
// DONE 2024/5/24: 文件服务器在外网不稳定,增加重试机制
fileInfo.setDownloadSuccess(false);
downLoadFileInfo.setDownloadSuccess(false);
if (retryNum > 0) {
log.error("[GFS Download] 文件下载失败,重试中: {}", fileOut);
return download0(fileInfo, retryNum - 1);
return download0(downLoadFileInfo, retryNum - 1);
} else {
e.printStackTrace();
log.error("[GFS Download] 文件下载失败: {}", fileOut);
}
}
return fileInfo;
return downLoadFileInfo;
}
/** 简单校验验证文件完整性,并生成索引文件*/
@ -101,14 +101,14 @@ public class GfsDownloader extends BaseDataDownloader {
}
}
@Override
public List<FileInfo> downloadAll(List<FileInfo> fileInfoList) {
log.info("[GFS Download] 下载任务启动,共 {} 个文件", fileInfoList.size());
List<Future<FileInfo>> futures = new ArrayList<>();
List<FileInfo> finishList = new ArrayList<>();
for (FileInfo fileInfo : fileInfoList) {
futures.add(executorService.submit(() -> download(fileInfo)));
public List<DownLoadFileInfo> downloadAll(List<DownLoadFileInfo> downLoadFileInfoList) {
log.info("[GFS Download] 下载任务启动,共 {} 个文件", downLoadFileInfoList.size());
List<Future<DownLoadFileInfo>> futures = new ArrayList<>();
List<DownLoadFileInfo> finishList = new ArrayList<>();
for (DownLoadFileInfo downLoadFileInfo : downLoadFileInfoList) {
futures.add(executorService.submit(() -> download(downLoadFileInfo)));
}
for (Future<FileInfo> future : futures) {
for (Future<DownLoadFileInfo> future : futures) {
try {
finishList.add(future.get());
} catch (InterruptedException | ExecutionException e) {
@ -120,7 +120,7 @@ public class GfsDownloader extends BaseDataDownloader {
}
@Override
public List<FileInfo> getFilesInfo() {
public List<DownLoadFileInfo> getFilesInfo() {
// 时间选项填充
String lonlatBoxStr = null;
String levelsStr = null;
@ -145,28 +145,28 @@ public class GfsDownloader extends BaseDataDownloader {
// 分辨率为3h整数截断包括目标时刻本身
final int nFiles = 1 + gfsDataConfig.getDuration() / forecastStep;
// 存储文件信息
List<FileInfo> fileInfoList = new ArrayList<>(nFiles);
List<DownLoadFileInfo> downLoadFileInfoList = new ArrayList<>(nFiles);
for (int i = 0; i < nFiles; i++) {
FileInfo fileInfo = new FileInfo();
DownLoadFileInfo downLoadFileInfo = new DownLoadFileInfo();
int forecastHour = i * forecastStep + hourDiff;
fileInfo.setForecastHour(forecastHour);
downLoadFileInfo.setForecastHour(forecastHour);
String baseURL = getBaseURL(forecastHour, levelsStr, variablesStr, lonlatBoxStr);
fileInfo.setUrl(baseURL);
fileInfo.setRefTimeStr(refTimeStr);
fileInfo.setForecastUTCTimeStr(
this.getRefTime().plusHours(fileInfo.getForecastHour())
downLoadFileInfo.setUrl(baseURL);
downLoadFileInfo.setRefTimeStr(refTimeStr);
downLoadFileInfo.setForecastUTCTimeStr(
this.getRefTime().plusHours(downLoadFileInfo.getForecastHour())
.format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING))
);
fileInfo.setForecastBJTimeStr(
DateTimeUtils.getLocalZoneDateTime(this.getRefTime().plusHours(fileInfo.getForecastHour()))
downLoadFileInfo.setForecastBJTimeStr(
DateTimeUtils.getLocalZoneDateTime(this.getRefTime().plusHours(downLoadFileInfo.getForecastHour()))
.format(DateTimeFormatter.ofPattern(Constant.BJT_TIME_STRING))
);
fileInfo.setFileName(String.format("%s-from-%s+%d.grib2", fileInfo.getForecastBJTimeStr(), fileInfo.getRefTimeStr(), forecastHour));
fileInfo.setSavePath(savePath);
fileInfoList.add(fileInfo);
downLoadFileInfo.setFileName(String.format("%s-from-%s+%d.grib2", downLoadFileInfo.getForecastBJTimeStr(), downLoadFileInfo.getRefTimeStr(), forecastHour));
downLoadFileInfo.setSavePath(savePath);
downLoadFileInfoList.add(downLoadFileInfo);
}
return fileInfoList;
return downLoadFileInfoList;
}

@ -43,8 +43,8 @@ public enum GfsVariableIsobaricEnum {
public static String getGfsVariableName(GfsVariableIsobaricEnum gfsVariableIsobaricEnum) {
return variableName.get(gfsVariableIsobaricEnum.nameInApi);
}
public static String getGfsVariableName(String tableVariableName) {
return variableName.get(tableVariableName);
public static String getGfsVariableName(String nameInApi) {
return variableName.get(nameInApi);
}
public static String getVariableNameInApi(String nameInFile) {

@ -5,7 +5,6 @@ import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.StoreOptions;
import com.aliyun.tablestore.grid.model.grid.Grid2D;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
@ -33,7 +32,6 @@ import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
@ -52,7 +50,7 @@ public class GfsDataImport extends BaseTableOperation {
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
/**
* init meta data to table store.
*
*
* @param dataSetID
* @param dataType
@ -81,7 +79,7 @@ public class GfsDataImport extends BaseTableOperation {
}
/**
* update meta and set status to DONE when data import finished.
* meta
*
* @param meta
* @return
@ -92,31 +90,34 @@ public class GfsDataImport extends BaseTableOperation {
return meta;
}
/**
* meta
*/
public GridDataSetMeta putMeta(GridDataSetMeta meta) throws Exception {
tableStoreGrid.putDataSetMeta(meta);
return meta;
}
/** 导入指定起报时间的所有文件*/
public List<ImportResult> importData(OffsetDateTime refTime) throws Exception {
public synchronized List<ImportFileInfo> importData(OffsetDateTime refTime) throws Exception {
importing = true;
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES);
}
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
// datasetId和起报时间绑定
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
try {
long start = System.currentTimeMillis();
List<String> fileList = getFiles(refTime);
if (CollectionUtils.isEmpty(fileList)) {
throw new AppException(ErrorCode.NO_NC_OR_GRIB_FILES);
}
log.info("[tablestore] 数据导入开始, refTime = {}...", refTime);
// datasetId和起报时间绑定
String dataSetId = refTime.format(DateTimeFormatter.ofPattern(Constant.DATA_SET_ID_FORMAT_STRING));
List<String> fileVariables = getFileVariables(tableConfig.variableList);
int[] shape = new int[]{tableConfig.timeSizeMax, tableConfig.levSize, tableConfig.latSize, tableConfig.lonSize};
GridDataSetMeta meta = initMeta(dataSetId, tableConfig.dataType, fileVariables, shape);
List<String> forecastHours = new ArrayList<>(); // 所有有效预报时效坐标记录到数据库属性中
// todo 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
List<ImportResult> finishedList;
// List<Future<ImportResult>> futures = new ArrayList<>();
// todo 2024/5/13: 待优化,用于数据库的索引,必须连续,因此必须保证前一个时刻成功导入后才能导入下一个时刻,数据量大的时候导入时间较长
List<ImportFileInfo> finishedList;
// List<Future<ImportResult>> futures = new ArrayList<>();
finishedList = new ArrayList<>();
for (int i = 0; i < fileList.size(); i++) {
String file = fileList.get(i);
@ -124,37 +125,39 @@ public class GfsDataImport extends BaseTableOperation {
int forecastHour = getForecastHourFromFilename(file);
// 2024/5/13:使用多线程并使用Future来获取结果但是实际瓶颈可能是网络带宽而非计算
// futures.add(executorService.submit(() -> importFromNcFile(meta, file, iTime,forecastHour)));
ImportResult importResult = importFromNcFile(meta, file, iTime, forecastHour);
finishedList.add(importResult);
if (importResult.isSuccess()) {
forecastHours.add(String.valueOf(importResult.getForecastHour()));
ImportFileInfo importFileInfo = importFromNcFile(meta, file, iTime, forecastHour);
finishedList.add(importFileInfo);
if (importFileInfo.isSuccess()) {
forecastHours.add(String.valueOf(importFileInfo.getForecastHour()));
}
}
// for (Future<ImportResult> future : futures) {
// ImportResult importResult = future.get();
// finishedList.add(importResult);
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// if (importResult.isSuccess()) {
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// }
// }
// for (Future<ImportResult> future : futures) {
// ImportResult importResult = future.get();
// finishedList.add(importResult);
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// if (importResult.isSuccess()) {
// forecastHours.add(String.valueOf(importResult.getForecastHour()));
// }
// }
List<ImportResult> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.warn("存在气象数据导入数据库失败,失败文件列表: {} ", failedList);
meta.addAttribute("status", "WRONG");
} else {
meta.addAttribute("status", "DONE");
List<ImportFileInfo> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.warn("存在气象数据导入数据库失败,失败文件列表: {} ", failedList);
meta.addAttribute("status", "WRONG");
} else {
meta.addAttribute("status", "DONE");
}
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
gfsDataFetcher.lastGridDataSetMeta = meta;
log.info("[tablestore]: 更新最新的数据元信息: {}", meta);
return finishedList;
} finally {
importing = false;
}
long end = System.currentTimeMillis();
log.info("[tablestore] 数据导入完成, 耗时: {} s, forecastHours: {}", (end - start)/1000., forecastHours);
meta.setForecastHours(forecastHours);
meta.addAttribute("reference_time", refTime.toInstant().toEpochMilli());
putMeta(meta);
gfsDataFetcher.lastGridDataSetMeta = meta;
log.info("[tablestore]: 更新最新的数据元信息: {}", meta);
importing = false;
return finishedList;
}
/**
@ -164,7 +167,7 @@ public class GfsDataImport extends BaseTableOperation {
* @param iTime
* @param forecastHour
*/
public ImportResult importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
public ImportFileInfo importFromNcFile(GridDataSetMeta meta, String file, int iTime, int forecastHour) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
@ -176,6 +179,7 @@ public class GfsDataImport extends BaseTableOperation {
int ysize = meta.getySize();
for (String variableName : meta.getVariables()) {
// 风速在最后单独处理
if ("Wind_speed_gust_surface".equals(variableName) || "Wind_speed_isobaric".equals(variableName) || "Wind_direction_isobaric".equals(variableName)
|| "Wind_speed_height_above_ground".equals(variableName) || "Wind_direction_height_above_ground".equals(variableName)) {
continue;
@ -183,7 +187,7 @@ public class GfsDataImport extends BaseTableOperation {
Variable variable = ncFile.findVariable(variableName);
if (variable != null) {
int shapeLength = variable.getShape().length;
if (shapeLength== 4) {
if (shapeLength== 4) { // 存在垂直坐标,即变量随高度变化
// 气压坐标和高度坐标保持各自的索引
int zSize = variable.getShape(1);
for (int z = 0; z < zSize; z++) {
@ -195,7 +199,7 @@ public class GfsDataImport extends BaseTableOperation {
writer.writeGrid2D(variableName, iTime, z, grid2D);
}
} else if (shapeLength == 3) {
// DONE 2024/5/24: 导入不含高度坐标的数据
// DONE 2024/5/24: 导入不含高度坐标的数据,如降水量,阵风风速
Array array = variable.read(new int[]{0, 0, 0}, new int[]{1,xsize, ysize});
transferUnit(variableName, array);
Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
@ -210,12 +214,13 @@ public class GfsDataImport extends BaseTableOperation {
importWind(writer, meta, ncFile, iTime);
} catch (Exception e) {
log.error("[tablestore] 导入文件数据失败,_t={}: {}", forecastHour, file, e);
return new ImportResult(false, file, forecastHour, iTime);
return new ImportFileInfo(false, file, forecastHour, iTime);
}
log.info("[tablestore] 导入文件数据成功,_t={}: {}", forecastHour, file);
return new ImportResult(true, file, forecastHour, iTime);
return new ImportFileInfo(true, file, forecastHour, iTime);
}
/**单位转换*/
private void transferUnit(String variableName, Array array) {
if (GfsVariableIsobaricEnum.TEMP.getNameInFile().equals(variableName)) {
MeteoUtils.kelvin2Celsius(array);
@ -322,13 +327,13 @@ public class GfsDataImport extends BaseTableOperation {
}
@Data
public static class ImportResult {
public static class ImportFileInfo {
private boolean success;
private String file;
private int forecastHour;
private int iTime;
public ImportResult(boolean success, String file, int forecastHour, int iTime) {
public ImportFileInfo(boolean success, String file, int forecastHour, int iTime) {
this.success = success;
this.file = file;
this.forecastHour = forecastHour;

@ -1,11 +1,10 @@
package com.htfp.weather.schedule;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.DownLoadFileInfo;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportFileInfo;
import com.htfp.weather.info.Constant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
@ -60,16 +59,13 @@ public class GridDataProcessor {
clearExpiredData();
}
public boolean download() {
private boolean download() {
gfsDownloader.iniTimeSetting();
try {
log.info("[schedule-start] 开始下载气象数据");
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
// if (test){
// throw new RuntimeException("测试异常");
// }
List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo();
List<DownLoadFileInfo> finishedList = gfsDownloader.downloadAll(downLoadFileInfoList);
List<DownLoadFileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule] 下载失败文件列表: {}", failedList);
downloadFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
@ -84,16 +80,13 @@ public class GridDataProcessor {
}
}
public boolean importToTableStore(){
private boolean importToTableStore(){
try {
OffsetDateTime refTime = gfsDownloader.getRefTime();
log.info("[schedule-start] 开始将数据导入数据库, 起报时间: {}", refTime);
// if (test){
// throw new RuntimeException("测试异常");
// }
List<ImportResult> importResults = gfsDataImport.importData(refTime);
List<ImportResult> finishedList = importResults.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<ImportResult> failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
List<ImportFileInfo> importFileInfos = gfsDataImport.importData(refTime);
List<ImportFileInfo> finishedList = importFileInfos.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<ImportFileInfo> failedList = importFileInfos.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule-end] 气象数据导入数据库失败, 文件列表: {} ", failedList);
importFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
@ -132,7 +125,7 @@ public class GridDataProcessor {
log.info("[schedule-end] 过期数据清理结束");
}
private void downloadFailedReport(String refTime, List<FileInfo> finishedList, List<FileInfo> failedList) {
private void downloadFailedReport(String refTime, List<DownLoadFileInfo> finishedList, List<DownLoadFileInfo> failedList) {
// 邮件
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
@ -185,7 +178,7 @@ public class GridDataProcessor {
}
}
private void importFailedReport(String refTime, List<ImportResult> finishedList, List<ImportResult> failedList) {
private void importFailedReport(String refTime, List<ImportFileInfo> finishedList, List<ImportFileInfo> failedList) {
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据入库失败通知");
mail.setTos(mailReceiver);

@ -1,20 +1,23 @@
package com.htfp.weather.web.controller;
import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.DownLoadFileInfo;
import com.htfp.weather.download.gfs.GfsDataConfig;
import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportFileInfo;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.web.exception.AppException;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.request.GfsConfigUpdate;
import com.htfp.weather.web.pojo.request.TableConfigUpdate;
import com.htfp.weather.web.pojo.response.DownloadResult;
import com.htfp.weather.web.pojo.response.ImportResult;
import com.htfp.weather.web.pojo.response.Result;
import com.htfp.weather.web.service.FileService;
import com.htfp.weather.web.service.GfsDataServiceImpl;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -23,11 +26,11 @@ import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @Author : shiyi
@ -43,11 +46,11 @@ public class ConfigController {
@Resource
GfsDataConfig gfsDataConfig;
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataImport gfsDataImport;
@Resource
FileService fileService;
@Resource
GfsDataServiceImpl gfsDataService;
private final String SECRET = "htfpweather";
private void validSecret(String secret) {
@ -121,14 +124,8 @@ public class ConfigController {
validSecret(secret);
// List<FileInfo> allCompleted = false;
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
List<FileInfo> successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList());
List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
Map<String, List<FileInfo>> data = new HashMap<>();
data.put("successList", successList);
data.put("failedList", failedList);
DownloadResult data = gfsDataService.downloadNowAllFile();
List<DownLoadFileInfo> failedList = data.getFailedList();
if (CollectionUtils.isEmpty(failedList)) {
return Result.success(data);
} else {
@ -147,24 +144,8 @@ public class ConfigController {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime offsetDateTime = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING));
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo(); // TODO 2024/6/12: 可以优化不用生成24小时所有的文件信息
for (FileInfo fileInfo : fileInfoList) {
if (fileInfo.getForecastUTCTimeStr().equals(targetUtcStr)) {
FileInfo downloadResult = gfsDownloader.download(fileInfo);
if (downloadResult.isDownloadSuccess()) {
return Result.success(downloadResult);
} else {
return Result.error(ErrorCode.DOWNLOAD_ERROR);
}
}
}
return Result.error(ErrorCode.QUERY_TIME_ERROR, ": 请选择当前时间之后24小时范围内的时刻");
} catch (AppException e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
DownLoadFileInfo downLoadFileInfo = gfsDataService.downloadSingleFile(offsetDateTime);
return Result.success(downLoadFileInfo);
}
/**
@ -175,17 +156,34 @@ public class ConfigController {
String secret = params.get("secret");
validSecret(secret);
OffsetDateTime time = OffsetDateTime.parse(params.get("time"), Constant.API_TIME_FORMATTER);
return getImportResponse(time);
}
/**
*
*/
@PostMapping("/downloadAndImport")
public Result downloadAndImport(@RequestBody Map<String, String> params) {
Result downloadResponse = downloadAllFiles(params);
if (downloadResponse.isSuccess()) {
DownloadResult data = (DownloadResult)downloadResponse.getData();
String refTimeStr = data.getSuccessList().get(0).getRefTimeStr();
OffsetDateTime time = OffsetDateTime.of(LocalDateTime.parse(refTimeStr, DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING)), ZoneOffset.UTC);
return getImportResponse(time);
} else {
return downloadResponse;
}
}
@NotNull
private Result getImportResponse(OffsetDateTime time) {
try {
List<ImportResult> finishedList = gfsDataImport.importData(time);
List<ImportResult> successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<ImportResult> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
Map<String, List<ImportResult>> data = new HashMap<>();
data.put("successList", successList);
data.put("failedList", failedList);
ImportResult importResult = gfsDataService.importData(time);
List<ImportFileInfo> failedList = importResult.getFailedList();
if (CollectionUtils.isEmpty(failedList)) {
return Result.success(data);
return Result.success(importResult);
} else {
return new Result(false, ErrorCode.TABLESTORE_IMPORT_ERROR.getCode(), ErrorCode.TABLESTORE_IMPORT_ERROR.getMsg() + ": 数据导入未全部完成", data);
return new Result(false, ErrorCode.TABLESTORE_IMPORT_ERROR.getCode(), ErrorCode.TABLESTORE_IMPORT_ERROR.getMsg() + ": 数据导入未全部完成", importResult);
}
} catch (AppException e) {
return Result.error(e.getErrorCode());

@ -0,0 +1,17 @@
package com.htfp.weather.web.pojo.response;
import com.htfp.weather.download.DownLoadFileInfo;
import lombok.Data;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/7/3 11:37
* @Description :
*/
@Data
public class DownloadResult {
List<DownLoadFileInfo> successList;
List<DownLoadFileInfo> failedList;
}

@ -0,0 +1,18 @@
package com.htfp.weather.web.pojo.response;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportFileInfo;
import lombok.Data;
import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/7/3 11:38
* @Description :
*/
@Data
public class ImportResult {
List<ImportFileInfo> successList;
List<ImportFileInfo> failedList;
}

@ -14,7 +14,7 @@ import java.util.List;
/**
* @Author : shiyi
* @Date : 2024/6/12 11:25
* @Description :
* @Description :
*/
@Component
public class FileService {
@ -57,13 +57,7 @@ public class FileService {
}
return subDirList;
}
public void uploadFile(String filePath){
// 文件上传逻辑
}
public void downloadFile(String filePath){
// 文件下载逻辑
}
public void deleteFile(File file) throws IOException {
if (file == null || !file.exists()) {

@ -4,12 +4,15 @@ import com.aliyun.tablestore.grid.consts.AttributionEnum;
import com.aliyun.tablestore.grid.model.GridDataSet;
import com.aliyun.tablestore.grid.model.GridDataSetMeta;
import com.aliyun.tablestore.grid.model.grid.Grid4D;
import com.htfp.weather.download.DownLoadFileInfo;
import com.htfp.weather.download.gfs.GfsDownloader;
import com.htfp.weather.download.gfs.GfsLevelsEnum;
import com.htfp.weather.download.gfs.GfsVariableHeightEnum;
import com.htfp.weather.download.gfs.GfsVariableIsobaricEnum;
import com.htfp.weather.griddata.common.TableConfig;
import com.htfp.weather.griddata.common.ValueRange;
import com.htfp.weather.griddata.operation.GfsDataFetcher;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.info.Constant;
import com.htfp.weather.utils.DateTimeUtils;
import com.htfp.weather.utils.MeteoUtils;
@ -27,7 +30,9 @@ import java.lang.reflect.Method;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Author : shiyi
@ -41,9 +46,60 @@ public class GfsDataServiceImpl implements IDataService {
GfsDataFetcher gfsDataFetcher;
@Resource
TableConfig tableConfig;
@Resource
GfsDownloader gfsDownloader;
@Resource
GfsDataImport gfsDataImport;
// 下载和导入数据
public DownloadResult downloadNowAllFile() {
gfsDownloader.iniTimeSetting();
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo();
List<DownLoadFileInfo> finishedList = gfsDownloader.downloadAll(downLoadFileInfoList);
List<DownLoadFileInfo> successList = finishedList.stream().filter(DownLoadFileInfo::isDownloadSuccess).collect(Collectors.toList());
List<DownLoadFileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
DownloadResult downloadResult = new DownloadResult();
downloadResult.setSuccessList(successList);
downloadResult.setFailedList(failedList);
return downloadResult;
}
public DownLoadFileInfo downloadSingleFile(OffsetDateTime offsetDateTime) {
String targetUtcStr = DateTimeUtils.getUTCDateTime(offsetDateTime).format(DateTimeFormatter.ofPattern(Constant.UTC_TIME_STRING));
try {
gfsDownloader.iniTimeSetting();
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo(); // TODO 2024/6/12: 可以优化不用生成24小时所有的文件信息
for (DownLoadFileInfo downLoadFileInfo : downLoadFileInfoList) {
if (downLoadFileInfo.getForecastUTCTimeStr().equals(targetUtcStr)) {
DownLoadFileInfo downloadResult = gfsDownloader.download(downLoadFileInfo);
if (downloadResult.isDownloadSuccess()) {
return downloadResult;
} else {
throw new AppException(ErrorCode.DOWNLOAD_ERROR);
}
}
}
throw new AppException(ErrorCode.QUERY_TIME_ERROR, ": 请选择当前时间之后24小时范围内的时刻");
} catch (Exception e) {
throw new AppException(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
}
public ImportResult importData(OffsetDateTime time) throws Exception {
List<GfsDataImport.ImportFileInfo> finishedList = gfsDataImport.importData(time);
List<GfsDataImport.ImportFileInfo> successList = finishedList.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<GfsDataImport.ImportFileInfo> failedList = finishedList.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
ImportResult importResult = new ImportResult();
importResult.setFailedList(failedList);
importResult.setSuccessList(successList);
return importResult;
}
public void downloadAndImportData() {
}
// 从tablestore获取数据
/**
*
*
@ -447,8 +503,7 @@ public class GfsDataServiceImpl implements IDataService {
return result;
}
@NotNull
private static String getVariableNameInApi(GfsLevelsEnum levelFlag, Map.Entry<String, Grid4D> variable) {
private String getVariableNameInApi(GfsLevelsEnum levelFlag, Map.Entry<String, Grid4D> variable) {
String variableNameInFile = variable.getKey();
String variableNameInApi;
if (GfsLevelsEnum.SURFACE.equals(levelFlag)) {

@ -56,8 +56,12 @@ public class CmaServiceImpl implements ISurfaceDataService{
return getSurfaceWarningByCounty(countyId);
}
public List<SurfaceWeatherWarning> getSurfaceWarningByCounty(String countyCode) throws Exception {
return warningCache.get(countyCode);
public List<SurfaceWeatherWarning> getSurfaceWarningByCounty(String countyCode) {
if (warningCache.containsKey(countyCode)) {
return warningCache.get(countyCode);
} else {
return new ArrayList<>();
}
// else {
// // 如果缓存中没有,则先查一遍
// updateCmaWarning(countyCode);

@ -28,24 +28,24 @@ class GfsDownloaderTest {
@Test
void download() {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.download(fileInfoList.get(0));
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.download(downLoadFileInfoList.get(0));
}
@Test
void downloadAll() {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.downloadAll(fileInfoList);
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.downloadAll(downLoadFileInfoList);
}
@Test
void getFilesInfo() {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
for (FileInfo fileInfo :fileInfoList) {
System.out.println(fileInfo);
List<DownLoadFileInfo> downLoadFileInfoList = gfsDownloader.getFilesInfo();
for (DownLoadFileInfo downLoadFileInfo : downLoadFileInfoList) {
System.out.println(downLoadFileInfo);
}
}

Loading…
Cancel
Save