From 7edc89952763b05d39a3b1d0200c3174db4b1988 Mon Sep 17 00:00:00 2001 From: shiyi Date: Mon, 3 Jun 2024 13:40:10 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=A4=B1=E8=B4=A5=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E3=80=81=E8=BF=94=E5=9B=9E=E6=88=90=E5=8A=9F=E5=92=8C?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E7=9A=84=E6=95=B0=E6=8D=AE=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../weather/download/BaseDataDownloader.java | 4 +- .../htfp/weather/download/GfsDownloader.java | 71 ++++++++++--------- .../weather/schedule/GridDataProcesser.java | 5 +- .../web/controller/ConfigController.java | 18 ++++- .../htfp/weather/web/exception/ErrorCode.java | 1 + 5 files changed, 60 insertions(+), 39 deletions(-) diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java index 3537993..73fb052 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java +++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java @@ -32,7 +32,7 @@ public abstract class BaseDataDownloader { * @return 下载成功/失败 * @throws IOException */ - public abstract boolean download(FileInfo fileInfo) throws IOException; + public abstract FileInfo download(FileInfo fileInfo) throws IOException; /** * 下载所有目标文件 @@ -40,7 +40,7 @@ public abstract class BaseDataDownloader { * @param filesInfo * @return */ - public abstract boolean downloadAll(List fileInfoList); + public abstract List downloadAll(List fileInfoList); public BaseDataDownloader() { diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java index 73344a8..b17297a 100644 --- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java +++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java @@ -13,8 +13,12 @@ import ucar.nc2.NetcdfFiles; import javax.annotation.Resource; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.net.ConnectException; import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.*; @@ -62,37 +66,40 @@ public class GfsDownloader extends BaseDataDownloader { } @Override - public boolean download(FileInfo fileInfo) { + public FileInfo download(FileInfo fileInfo) { + return download0(fileInfo, 2); + } + private FileInfo download0(FileInfo fileInfo, int retryNum) { + + String url = fileInfo.getUrl(); + File destDir = new File(fileInfo.getSavePath()); + if (!destDir.exists() && !destDir.mkdirs()) { + throw new RuntimeException("创建文件夹" + destDir + "失败"); + } + File fileOut = new File(destDir, fileInfo.getFileName()); + log.info("[GFS Download] 文件下载中，保存至 {}", fileOut); + try { - String url = fileInfo.getUrl(); - File destDir = new File(fileInfo.getSavePath()); - if (!destDir.exists()) { - destDir.mkdirs(); - } - File fileOut = new File(destDir, fileInfo.getFileName()); - // if (fileOut.exists()) { - // log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath()); - // return true; + // TODO 2024/5/24: 增加重试机制 + // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) { + // throw new RuntimeException(); // } - log.info("[GFS Download] 文件下载中，保存至 {}", fileOut); - // FileOutputStream fos = new FileOutputStream(filePath); - // try { - // ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream()); - // fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - // } catch (ConnectException e) { - // log.error("连接超时:{}", url); - // } - // - // fos.flush(); - // fos.close(); - FileUtils.copyURLToFile(new URL(url), fileOut); + FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000); log.info("[GFS Download] 文件下载成功: {}", fileOut); - return fileValid(fileOut.getAbsolutePath()); + fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath())); } catch (Exception e) { - log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo); - return false; + fileInfo.setDownloadSuccess(false); + if (retryNum > 0) { + log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut); + return download0(fileInfo, retryNum - 1); + } else { + e.printStackTrace(); + log.error("[GFS Download] 文件下载失败: {}", fileOut); + } } + return fileInfo; } + private boolean fileValid(String file) { try (NetcdfFile ncFile = NetcdfFiles.open(file)) { ncFile.getLocation(); @@ -102,22 +109,22 @@ public class GfsDownloader extends BaseDataDownloader { } } @Override - public boolean downloadAll(List fileInfoList) { + public List downloadAll(List fileInfoList) { log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size()); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); + List finishList = new ArrayList<>(); for (FileInfo fileInfo : fileInfoList) { futures.add(executorService.submit(() -> download(fileInfo))); } - boolean allCompleted = true; - for (Future future : futures) { + for (Future future : futures) { try { - allCompleted = allCompleted && future.get(); + finishList.add(future.get()); } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } } - log.info("[GFS Download] 下载完成，共 {} 个文件", fileInfoList.size()); - return allCompleted; + log.info("[GFS Download] 下载完成，共 {} 个文件", finishList.size()); + return finishList; } @Override diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java index 0260f94..87bb980 100644 --- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java +++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java @@ -42,13 +42,12 @@ public class GridDataProcesser { gfsDownloader.iniTimeSetting(); OffsetDateTime refTime = gfsDownloader.getRefTime(); List fileInfoList = gfsDownloader.getFilesInfo(); - boolean allComplete = gfsDownloader.downloadAll(fileInfoList); - if (allComplete) { + // if (allComplete) { try { gfsDataImport.importData(refTime); } catch (AppExcpetion e) { log.info("导入数据失败"); - } + // } } } diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java index 792fda6..bb68bf5 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java +++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java @@ -8,14 +8,18 @@ import com.htfp.weather.griddata.operation.GfsDataImport; import com.htfp.weather.web.exception.AppExcpetion; import com.htfp.weather.web.exception.ErrorCode; import com.htfp.weather.web.pojo.response.Result; +import org.springframework.util.CollectionUtils; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import java.io.IOException; import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * @Author : shiyi @@ -85,11 +89,21 @@ public class ConfigController { @RequestMapping("/download") public Result download(@RequestParam String secret) { validSecret(secret); + // List allCompleted = false; try { gfsDownloader.iniTimeSetting(); List fileInfoList = gfsDownloader.getFilesInfo(); - gfsDownloader.downloadAll(fileInfoList); - return Result.success(fileInfoList); + List finishedList = gfsDownloader.downloadAll(fileInfoList); + List successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList()); + List failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList()); + Map> data = new HashMap<>(); + data.put("successList", successList); + data.put("failedList", failedList); + if (CollectionUtils.isEmpty(failedList)) { + return Result.success(data); + } else { + return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data); + } } catch (Exception e) { return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage()); } diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java index cd1cedc..7efa8e4 100644 --- a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java +++ b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java @@ -12,6 +12,7 @@ public enum ErrorCode { CONFIG_ERROR(1002, "配置相关错误 "), SECRET_ERROR(1003, "无权限访问"), DOWNLOAD_START_ERROR(1004, "数据下载启动错误"), + DOWNLOAD_ERROR(1004, "数据下载错误"), HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"), HE_FENG_REQUEST_ERROR(3002, "查询请求错误"), CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),