504-Software-Development-Group
/
WeatherService
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

下载失败重试、返回成功和失败的数据信息

Browse Source
refactor
shiyi 10 months ago
parent c950750e67
commit 7edc899527
5 changed files with 60 additions and 39 deletions
Show Stats Download Patch File Download Diff File

4
weather-service/src/main/java/com/htfp/weather/download/BaseDataDownloader.java
Unescape Escape View File

@ -32,7 +32,7 @@ public abstract class BaseDataDownloader {
* @return 下载成功/失败
* @throws IOException
*/
public abstract boolean download(FileInfo fileInfo) throws IOException;
public abstract FileInfo download(FileInfo fileInfo) throws IOException;
/**
* 下载所有目标文件
@ -40,7 +40,7 @@ public abstract class BaseDataDownloader {
* @param filesInfo
* @return
*/
public abstract boolean downloadAll(List<FileInfo> fileInfoList);
public abstract List<FileInfo> downloadAll(List<FileInfo> fileInfoList);
public BaseDataDownloader() {

63
weather-service/src/main/java/com/htfp/weather/download/GfsDownloader.java
Unescape Escape View File

@ -13,8 +13,12 @@ import ucar.nc2.NetcdfFiles;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.ConnectException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -62,37 +66,40 @@ public class GfsDownloader extends BaseDataDownloader {
}
@Override
public boolean download(FileInfo fileInfo) {
try {
public FileInfo download(FileInfo fileInfo) {
return download0(fileInfo, 2);
}
private FileInfo download0(FileInfo fileInfo, int retryNum) {
String url = fileInfo.getUrl();
File destDir = new File(fileInfo.getSavePath());
if (!destDir.exists()) {
destDir.mkdirs();
if (!destDir.exists() && !destDir.mkdirs()) {
throw new RuntimeException("创建文件夹" + destDir + "失败");
}
File fileOut = new File(destDir, fileInfo.getFileName());
// if (fileOut.exists()) {
// log.info("[GFS Download] 文件已存在，忽略下载: {}", fileOut.getAbsolutePath());
// return true;
// }
log.info("[GFS Download] 文件下载中，保存至 {}", fileOut);
// FileOutputStream fos = new FileOutputStream(filePath);
// try {
// ReadableByteChannel rbc = Channels.newChannel(new URL(url).openStream());
// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// } catch (ConnectException e) {
// log.error("连接超时:{}", url);
try {
// TODO 2024/5/24: 增加重试机制
// if (fileInfo.getForecastHour() == 12 || fileInfo.getForecastHour() == 18 || fileInfo.getForecastHour() == 24) {
// throw new RuntimeException();
// }
//
// fos.flush();
// fos.close();
FileUtils.copyURLToFile(new URL(url), fileOut);
FileUtils.copyURLToFile(new URL(url), fileOut, 30000, 30000);
log.info("[GFS Download] 文件下载成功: {}", fileOut);
return fileValid(fileOut.getAbsolutePath());
fileInfo.setDownloadSuccess(fileValid(fileOut.getAbsolutePath()));
} catch (Exception e) {
log.error("[GFS Download]下载失败:{} {}", e.getMessage(), fileInfo);
return false;
fileInfo.setDownloadSuccess(false);
if (retryNum > 0) {
log.error("[GFS Download] 文件下载失败，重试中: {}", fileOut);
return download0(fileInfo, retryNum - 1);
} else {
e.printStackTrace();
log.error("[GFS Download] 文件下载失败: {}", fileOut);
}
}
return fileInfo;
}
private boolean fileValid(String file) {
try (NetcdfFile ncFile = NetcdfFiles.open(file)) {
ncFile.getLocation();
@ -102,22 +109,22 @@ public class GfsDownloader extends BaseDataDownloader {
}
}
@Override
public boolean downloadAll(List<FileInfo> fileInfoList) {
public List<FileInfo> downloadAll(List<FileInfo> fileInfoList) {
log.info("[GFS Download] 下载任务启动，共 {} 个文件", fileInfoList.size());
List<Future<Boolean>> futures = new ArrayList<>();
List<Future<FileInfo>> futures = new ArrayList<>();
List<FileInfo> finishList = new ArrayList<>();
for (FileInfo fileInfo : fileInfoList) {
futures.add(executorService.submit(() -> download(fileInfo)));
}
boolean allCompleted = true;
for (Future<Boolean> future : futures) {
for (Future<FileInfo> future : futures) {
try {
allCompleted = allCompleted && future.get();
finishList.add(future.get());
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
log.info("[GFS Download] 下载完成，共 {} 个文件", fileInfoList.size());
return allCompleted;
log.info("[GFS Download] 下载完成，共 {} 个文件", finishList.size());
return finishList;
}
@Override

5
weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcesser.java
Unescape Escape View File

@ -42,13 +42,12 @@ public class GridDataProcesser {
gfsDownloader.iniTimeSetting();
OffsetDateTime refTime = gfsDownloader.getRefTime();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
boolean allComplete = gfsDownloader.downloadAll(fileInfoList);
if (allComplete) {
// if (allComplete) {
try {
gfsDataImport.importData(refTime);
} catch (AppExcpetion e) {
log.info("导入数据失败");
}
// }
}
}

18
weather-service/src/main/java/com/htfp/weather/web/controller/ConfigController.java
Unescape Escape View File

@ -8,14 +8,18 @@ import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.web.exception.AppExcpetion;
import com.htfp.weather.web.exception.ErrorCode;
import com.htfp.weather.web.pojo.response.Result;
import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @Author : shiyi
@ -85,11 +89,21 @@ public class ConfigController {
@RequestMapping("/download")
public Result download(@RequestParam String secret) {
validSecret(secret);
// List<FileInfo> allCompleted = false;
try {
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
gfsDownloader.downloadAll(fileInfoList);
return Result.success(fileInfoList);
List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
List<FileInfo> successList = finishedList.stream().filter(FileInfo::isDownloadSuccess).collect(Collectors.toList());
List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
Map<String, List<FileInfo>> data = new HashMap<>();
data.put("successList", successList);
data.put("failedList", failedList);
if (CollectionUtils.isEmpty(failedList)) {
return Result.success(data);
} else {
return new Result(false, ErrorCode.DOWNLOAD_ERROR.getCode(), ErrorCode.DOWNLOAD_ERROR.getMsg() + ":下载未全部完成", data);
}
} catch (Exception e) {
return Result.error(ErrorCode.DOWNLOAD_START_ERROR, e.getMessage());
}

1
weather-service/src/main/java/com/htfp/weather/web/exception/ErrorCode.java
Unescape Escape View File

@ -12,6 +12,7 @@ public enum ErrorCode {
CONFIG_ERROR(1002, "配置相关错误 "),
SECRET_ERROR(1003, "无权限访问"),
DOWNLOAD_START_ERROR(1004, "数据下载启动错误"),
DOWNLOAD_ERROR(1004, "数据下载错误"),
HE_FENG_THIS_AREA_HAVE_NO_DATA(3001, "查询的数据或地区不存在"),
HE_FENG_REQUEST_ERROR(3002, "查询请求错误"),
CAI_YUN_REQUEST_ERROR(4002, "查询请求错误"),

Write Preview
Loading…
Cancel
Save