From 7edc89952763b05d39a3b1d0200c3174db4b1988 Mon Sep 17 00:00:00 2001
From: shiyi <shi_yee@outlook.com>
Date: Mon, 3 Jun 2024 13:40:10 +0800
Subject: [PATCH] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=A4=B1=E8=B4=A5=E9=87=8D?=
 =?UTF-8?q?=E8=AF=95=E3=80=81=E8=BF=94=E5=9B=9E=E6=88=90=E5=8A=9F=E5=92=8C?=
 =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E7=9A=84=E6=95=B0=E6=8D=AE=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../weather/download/BaseDataDownloader.java  |  4 +-
 .../htfp/weather/download/GfsDownloader.java  | 71 ++++++++++---------
 .../weather/schedule/GridDataProcesser.java   |  5 +-
 .../web/controller/ConfigController.java      | 18 ++++-
 .../htfp/weather/web/exception/ErrorCode.java |  1 +
 5 files changed, 60 insertions(+), 39 deletions(-)

diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
index 3537993..73fb052 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
@@ -32,7 +32,7 @@ public abstract class BaseDataDownloader {
      * @return 下载成功/失败
      * @throws IOException
      */
-    public abstract boolean download(FileInfo fileInfo) throws IOException;
+    public abstract FileInfo download(FileInfo fileInfo) throws IOException;
 
     /**
      * 下载所有目标文件
@@ -40,7 +40,7 @@ public abstract class BaseDataDownloader {
      * @param filesInfo
      * @return
      */
-    public abstract boolean downloadAll(List<FileInfo> fileInfoList);
+    public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
 
 
     public BaseDataDownloader() {
diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
index 73344a8..b17297a 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
@@ -13,8 +13,12 @@ import ucar.nc2.NetcdfFiles;
 
 import javax.annotation.Resource;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.ConnectException;
 import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.time.*;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -62,37 +66,40 @@ public class GfsDownloader extends BaseDataDownloader {
     }
 
     @Override
-    public boolean download(FileInfo fileInfo) {
+    public FileInfo download(FileInfo fileInfo) {
+        return download0(fileInfo, 2);
+    }
+    private FileInfo download0(FileInfo fileInfo, int retryNum) {
+
+        String url = fileInfo.getUrl();
+        File destDir = new File(fileInfo.getSavePath());
+        if (!destDir.exists() && !destDir.mkdirs()) {
+            throw new RuntimeException("创建文件夹" + destDir + "失败");
+        }
+        File fileOut = new File(destDir, fileInfo.getFileName());
+        log.info("[GFS Download] 文件下载中,保存至 {}", fileOut);
+
         try {
-            String url = fileInfo.getUrl();
-            File destDir = new File(fileInfo.getSavePath());
-            if (!destDir.exists()) {
-                destDir.mkdirs();
-            }
-            File fileOut = new File(destDir, fileInfo.getFileName());
-            // if (fileOut.exists()) {
-            //     log.info("[GFS Download] 文件已存在,忽略下载: {}", fileOut.getAbsolutePath());
-            //     return true;
+            // TODO 2024/5/24: 增加重试机制
+            // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
+            //     throw new RuntimeException();
             // }
-            log.info("[GFS Download] 文件下载中,保存至 {}", fileOut);
-            // FileOutputStream fos = new FileOutputStream(filePath);
-            // try {
-            //     ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream());
-            //     fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
-            // } catch (ConnectException e) {
-            //     log.error("连接超时:{}", url);
-            // }
-            //
-            // fos.flush();
-            // fos.close();
-            FileUtils.copyURLToFile(new URL(url), fileOut);
+            FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
             log.info("[GFS Download] 文件下载成功: {}", fileOut);
-            return fileValid(fileOut.getAbsolutePath());
+            fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
         } catch (Exception e) {
-            log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
-            return false;
+            fileInfo.setDownloadSuccess(false);
+            if (retryNum > 0) {
+                log.error("[GFS Download] 文件下载失败,重试中: {}", fileOut);
+                return download0(fileInfo, retryNum - 1);
+            } else {
+                e.printStackTrace();
+                log.error("[GFS Download] 文件下载失败: {}", fileOut);
+            }
         }
+        return fileInfo;
     }
+
     private boolean fileValid(String file) {
         try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
             ncFile.getLocation();
@@ -102,22 +109,22 @@ public class GfsDownloader extends BaseDataDownloader {
         }
     }
     @Override
-    public boolean downloadAll(List<FileInfo> fileInfoList) {
+    public List<FileInfo> downloadAll(List<FileInfo> fileInfoList) {
         log.info("[GFS Download] 下载任务启动,共 {} 个文件", fileInfoList.size());
-        List<Future<Boolean>> futures = new ArrayList<>();
+        List<Future<FileInfo>> futures = new ArrayList<>();
+        List<FileInfo> finishList = new ArrayList<>();
         for (FileInfo fileInfo : fileInfoList) {
             futures.add(executorService.submit(() -> download(fileInfo)));
         }
-        boolean allCompleted = true;
-        for (Future<Boolean> future : futures) {
+        for (Future<FileInfo> future : futures) {
             try {
-                allCompleted = allCompleted && future.get();
+                finishList.add(future.get());
             } catch (InterruptedException | ExecutionException e) {
                 e.printStackTrace();
             }
         }
-        log.info("[GFS Download] 下载完成,共 {} 个文件", fileInfoList.size());
-        return allCompleted;
+        log.info("[GFS Download] 下载完成,共 {} 个文件", finishList.size());
+        return finishList;
     }
 
     @Override
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
index 0260f94..87bb980 100644
--- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
@@ -42,13 +42,12 @@ public class GridDataProcesser {
         gfsDownloader.iniTimeSetting();
         OffsetDateTime refTime = gfsDownloader.getRefTime();
         List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
-        boolean allComplete = gfsDownloader.downloadAll(fileInfoList);
-        if (allComplete) {
+        // if (allComplete) {
             try {
                 gfsDataImport.importData(refTime);
             } catch (AppExcpetion e) {
                 log.info("导入数据失败");
-            }
+            // }
         }
     }
 
diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
index 792fda6..bb68bf5 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
@@ -8,14 +8,18 @@ import com.htfp.weather.griddata.operation.GfsDataImport;
 import com.htfp.weather.web.exception.AppExcpetion;
 import com.htfp.weather.web.exception.ErrorCode;
 import com.htfp.weather.web.pojo.response.Result;
+import org.springframework.util.CollectionUtils;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
 import java.io.IOException;
 import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * @Author : shiyi
@@ -85,11 +89,21 @@ public class ConfigController {
     @RequestMapping("/download")
     public Result download(@RequestParam String secret) {
         validSecret(secret);
+        // List<FileInfo> allCompleted = false;
         try {
             gfsDownloader.iniTimeSetting();
             List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
-            gfsDownloader.downloadAll(fileInfoList);
-            return Result.success(fileInfoList);
+            List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
+            List<FileInfo> successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList());
+            List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
+            Map<String, List<FileInfo>> data = new HashMap<>();
+            data.put("successList", successList);
+            data.put("failedList", failedList);
+            if (CollectionUtils.isEmpty(failedList)) {
+                return Result.success(data);
+            } else {
+                return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data);
+            }
         } catch (Exception e) {
             return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
         }
diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
index cd1cedc..7efa8e4 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
@@ -12,6 +12,7 @@ public enum ErrorCode {
     CONFIG_ERROR(1002, "配置相关错误 "),
     SECRET_ERROR(1003, "无权限访问"),
     DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
+    DOWNLOAD_ERROR(1004, "数据下载错误"),
     HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
     HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
     CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),