From 7edc89952763b05d39a3b1d0200c3174db4b1988 Mon Sep 17 00:00:00 2001
From: shiyi <shi_yee@outlook.com>
Date: Mon, 3 Jun 2024 13:40:10 +0800
Subject: [PATCH] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=A4=B1=E8=B4=A5=E9=87=8D?=
=?UTF-8?q?=E8=AF=95=E3=80=81=E8=BF=94=E5=9B=9E=E6=88=90=E5=8A=9F=E5=92=8C?=
=?UTF-8?q?=E5=A4=B1=E8=B4=A5=E7=9A=84=E6=95=B0=E6=8D=AE=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../weather/download/BaseDataDownloader.java | 4 +-
.../htfp/weather/download/GfsDownloader.java | 71 ++++++++++---------
.../weather/schedule/GridDataProcesser.java | 5 +-
.../web/controller/ConfigController.java | 18 ++++-
.../htfp/weather/web/exception/ErrorCode.java | 1 +
5 files changed, 60 insertions(+), 39 deletions(-)
diff --git a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
index 3537993..73fb052 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
@@ -32,7 +32,7 @@ public abstract class BaseDataDownloader {
* @return 下载成功/失败
* @throws IOException
*/
- public abstract boolean download(FileInfo fileInfo) throws IOException;
+ public abstract FileInfo download(FileInfo fileInfo) throws IOException;
/**
* 下载所有目标文件
@@ -40,7 +40,7 @@ public abstract class BaseDataDownloader {
* @param filesInfo
* @return
*/
- public abstract boolean downloadAll(List<FileInfo> fileInfoList);
+ public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
public BaseDataDownloader() {
diff --git a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
index 73344a8..b17297a 100644
--- a/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
+++ b/weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
@@ -13,8 +13,12 @@ import ucar.nc2.NetcdfFiles;
import javax.annotation.Resource;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.ConnectException;
import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
@@ -62,37 +66,40 @@ public class GfsDownloader extends BaseDataDownloader {
}
@Override
- public boolean download(FileInfo fileInfo) {
+ public FileInfo download(FileInfo fileInfo) {
+ return download0(fileInfo, 2);
+ }
+ private FileInfo download0(FileInfo fileInfo, int retryNum) {
+
+ String url = fileInfo.getUrl();
+ File destDir = new File(fileInfo.getSavePath());
+ if (!destDir.exists() && !destDir.mkdirs()) {
+ throw new RuntimeException("创建文件夹" + destDir + "失败");
+ }
+ File fileOut = new File(destDir, fileInfo.getFileName());
+ log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
+
try {
- String url = fileInfo.getUrl();
- File destDir = new File(fileInfo.getSavePath());
- if (!destDir.exists()) {
- destDir.mkdirs();
- }
- File fileOut = new File(destDir, fileInfo.getFileName());
- // if (fileOut.exists()) {
- // log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath());
- // return true;
+ // TODO 2024/5/24: 增加重试机制
+ // if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
+ // throw new RuntimeException();
// }
- log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
- // FileOutputStream fos = new FileOutputStream(filePath);
- // try {
- // ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream());
- // fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
- // } catch (ConnectException e) {
- // log.error("连接超时:{}", url);
- // }
- //
- // fos.flush();
- // fos.close();
- FileUtils.copyURLToFile(new URL(url), fileOut);
+ FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
log.info("[GFS Download] 文件下载成功: {}", fileOut);
- return fileValid(fileOut.getAbsolutePath());
+ fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
- log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
- return false;
+ fileInfo.setDownloadSuccess(false);
+ if (retryNum > 0) {
+ log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut);
+ return download0(fileInfo, retryNum - 1);
+ } else {
+ e.printStackTrace();
+ log.error("[GFS Download] 文件下载失败: {}", fileOut);
+ }
}
+ return fileInfo;
}
+
private boolean fileValid(String file) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
ncFile.getLocation();
@@ -102,22 +109,22 @@ public class GfsDownloader extends BaseDataDownloader {
}
}
@Override
- public boolean downloadAll(List<FileInfo> fileInfoList) {
+ public List<FileInfo> downloadAll(List<FileInfo> fileInfoList) {
log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size());
- List<Future<Boolean>> futures = new ArrayList<>();
+ List<Future<FileInfo>> futures = new ArrayList<>();
+ List<FileInfo> finishList = new ArrayList<>();
for (FileInfo fileInfo : fileInfoList) {
futures.add(executorService.submit(() -> download(fileInfo)));
}
- boolean allCompleted = true;
- for (Future<Boolean> future : futures) {
+ for (Future<FileInfo> future : futures) {
try {
- allCompleted = allCompleted && future.get();
+ finishList.add(future.get());
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
- log.info("[GFS Download] 下载完成，共 {} 个文件", fileInfoList.size());
- return allCompleted;
+ log.info("[GFS Download] 下载完成，共 {} 个文件", finishList.size());
+ return finishList;
}
@Override
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
index 0260f94..87bb980 100644
--- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
@@ -42,13 +42,12 @@ public class GridDataProcesser {
gfsDownloader.iniTimeSetting();
OffsetDateTime refTime = gfsDownloader.getRefTime();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
- boolean allComplete = gfsDownloader.downloadAll(fileInfoList);
- if (allComplete) {
+ // if (allComplete) {
try {
gfsDataImport.importData(refTime);
} catch (AppExcpetion e) {
log.info("导入数据失败");
- }
+ // }
}
}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
index 792fda6..bb68bf5 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
@@ -8,14 +8,18 @@ import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.Result;
+import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
import java.time.OffsetDateTime;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
/**
* @Author : shiyi
@@ -85,11 +89,21 @@ public class ConfigController {
@RequestMapping("/download")
public Result download(@RequestParam String secret) {
validSecret(secret);
+ // List<FileInfo> allCompleted = false;
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
- gfsDownloader.downloadAll(fileInfoList);
- return Result.success(fileInfoList);
+ List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
+ List<FileInfo> successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList());
+ List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
+ Map<String, List<FileInfo>> data = new HashMap<>();
+ data.put("successList", successList);
+ data.put("failedList", failedList);
+ if (CollectionUtils.isEmpty(failedList)) {
+ return Result.success(data);
+ } else {
+ return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data);
+ }
} catch (Exception e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}
diff --git a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
index cd1cedc..7efa8e4 100644
--- a/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
+++ b/weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
@@ -12,6 +12,7 @@ public enum ErrorCode {
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
+ DOWNLOAD_ERROR(1004, "数据下载错误"),
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),