下载失败重试、返回成功和失败的数据信息

weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java

@@ -32,7 +32,7 @@ public abstract class BaseDataDownloader {
      * @return 下载成功/失败
      * @throws IOException
      */
-    public abstract boolean download(FileInfo fileInfo) throws IOException;
+    public abstract FileInfo download(FileInfo fileInfo) throws IOException;
 
     /**
      * 下载所有目标文件
@@ -40,7 +40,7 @@ public abstract class BaseDataDownloader {
      * @param filesInfo
      * @return
      */
-    public abstract boolean downloadAll(List<FileInfo> fileInfoList);
+    public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
 
 
     public BaseDataDownloader() {

weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java

@@ -13,8 +13,12 @@ import ucar.nc2.NetcdfFiles;
 
 import javax.annotation.Resource;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.ConnectException;
 import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.time.*;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -62,37 +66,40 @@ public class GfsDownloader extends BaseDataDownloader {
     }
 
     @Override
-    public boolean download(FileInfo fileInfo) {
+    public FileInfo download(FileInfo fileInfo) {
-        try {
+        return download0(fileInfo, 2);
+    }
+    private FileInfo download0(FileInfo fileInfo, int retryNum) {
+
         String url = fileInfo.getUrl();
         File destDir = new File(fileInfo.getSavePath());
-            if (!destDir.exists()) {
+        if (!destDir.exists() && !destDir.mkdirs()) {
-                destDir.mkdirs();
+            throw new RuntimeException("创建文件夹" + destDir + "失败");
         }
         File fileOut = new File(destDir, fileInfo.getFileName());
-            // if (fileOut.exists()) {
-            //     log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath());
-            //     return true;
-            // }
         log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
-            // FileOutputStream fos = new FileOutputStream(filePath);
+
-            // try {
+        try {
-            //     ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream());
+            // TODO 2024/5/24: 增加重试机制
-            //     fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+            // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
-            // } catch (ConnectException e) {
+            //     throw new RuntimeException();
-            //     log.error("连接超时:{}", url);
             // }
-            //
+            FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
-            // fos.flush();
-            // fos.close();
-            FileUtils.copyURLToFile(new URL(url), fileOut);
             log.info("[GFS Download] 文件下载成功: {}", fileOut);
-            return fileValid(fileOut.getAbsolutePath());
+            fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
         } catch (Exception e) {
-            log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
+            fileInfo.setDownloadSuccess(false);
-            return false;
+            if (retryNum > 0) {
+                log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut);
+                return download0(fileInfo, retryNum - 1);
+            } else {
+                e.printStackTrace();
+                log.error("[GFS Download] 文件下载失败: {}", fileOut);
             }
         }
+        return fileInfo;
+    }
+
     private boolean fileValid(String file) {
         try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
             ncFile.getLocation();
@@ -102,22 +109,22 @@ public class GfsDownloader extends BaseDataDownloader {
         }
     }
     @Override
-    public boolean downloadAll(List<FileInfo> fileInfoList) {
+    public List<FileInfo> downloadAll(List<FileInfo> fileInfoList) {
         log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size());
-        List<Future<Boolean>> futures = new ArrayList<>();
+        List<Future<FileInfo>> futures = new ArrayList<>();
+        List<FileInfo> finishList = new ArrayList<>();
         for (FileInfo fileInfo : fileInfoList) {
             futures.add(executorService.submit(() -> download(fileInfo)));
         }
-        boolean allCompleted = true;
+        for (Future<FileInfo> future : futures) {
-        for (Future<Boolean> future : futures) {
             try {
-                allCompleted = allCompleted && future.get();
+                finishList.add(future.get());
             } catch (InterruptedException | ExecutionException e) {
                 e.printStackTrace();
             }
         }
-        log.info("[GFS Download] 下载完成，共 {} 个文件", fileInfoList.size());
+        log.info("[GFS Download] 下载完成，共 {} 个文件", finishList.size());
-        return allCompleted;
+        return finishList;
     }
 
     @Override

weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java

@@ -42,13 +42,12 @@ public class GridDataProcesser {
         gfsDownloader.iniTimeSetting();
         OffsetDateTime refTime = gfsDownloader.getRefTime();
         List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
-        boolean allComplete = gfsDownloader.downloadAll(fileInfoList);
+        // if (allComplete) {
-        if (allComplete) {
             try {
                 gfsDataImport.importData(refTime);
             } catch (AppExcpetion e) {
                 log.info("导入数据失败");
-            }
+            // }
         }
     }
 

weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java

@@ -8,14 +8,18 @@ import com.htfp.weather.griddata.operation.GfsDataImport;
 import com.htfp.weather.web.exception.AppExcpetion;
 import com.htfp.weather.web.exception.ErrorCode;
 import com.htfp.weather.web.pojo.response.Result;
+import org.springframework.util.CollectionUtils;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
 import java.io.IOException;
 import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * @Author : shiyi
@@ -85,11 +89,21 @@ public class ConfigController {
     @RequestMapping("/download")
     public Result download(@RequestParam String secret) {
         validSecret(secret);
+        // List<FileInfo> allCompleted = false;
         try {
             gfsDownloader.iniTimeSetting();
             List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
-            gfsDownloader.downloadAll(fileInfoList);
+            List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
-            return Result.success(fileInfoList);
+            List<FileInfo> successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList());
+            List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
+            Map<String, List<FileInfo>> data = new HashMap<>();
+            data.put("successList", successList);
+            data.put("failedList", failedList);
+            if (CollectionUtils.isEmpty(failedList)) {
+                return Result.success(data);
+            } else {
+                return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data);
+            }
         } catch (Exception e) {
             return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
         }

weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java

@@ -12,6 +12,7 @@ public enum ErrorCode {
     CONFIG_ERROR(1002, "配置相关错误 "),
     SECRET_ERROR(1003, "无权限访问"),
     DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
+    DOWNLOAD_ERROR(1004, "数据下载错误"),
     HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
     HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
     CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),