下载或导入数据失败时,向指定服务器报告错误,同时用邮件通知维护人员

refactor
shiyi 9 months ago
parent c160cca531
commit 4864bd2fbd

@ -71,6 +71,18 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-mail</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>

@ -2,11 +2,13 @@ package com.htfp.weather;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.retry.annotation.EnableRetry;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.util.TimeZone;
@EnableScheduling
@EnableRetry // 开启重试机制
@EnableScheduling // 开启定时任务
@SpringBootApplication
public class WeatherServiceApplication {

@ -0,0 +1,14 @@
package com.htfp.weather.schedule;
import lombok.Data;
/**
* @Author : shiyi
* @Date : 2024/6/13 15:32
* @Description :
*/
@Data
public class ErrorReport {
String time;
String message;
}

@ -0,0 +1,66 @@
package com.htfp.weather.schedule;
import com.htfp.weather.utils.HttpClientUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.mail.SimpleMailMessage;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
/**
* @Author : shiyi
* @Date : 2024/6/13 10:57
* @Description :
*/
@Service
@Slf4j
public class ErrorReportService {
@Autowired
private JavaMailSender mailSender;
@Value("${mail.send.from}")
public String sender;// 发送者
// 发送简单文本文件
@Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2))
public void sendSimpleEmail(final Mail mail) {
try {
SimpleMailMessage message = new SimpleMailMessage();
message.setFrom(sender);
message.setTo(mail.getTos());
message.setSubject(mail.getSubject());
message.setText(mail.getContent());
mailSender.send(message);
log.info("[error report] 异常邮件发送成功,{} --> {}, 主题:{}", sender, mail.getTos(), mail.getSubject());
} catch (Exception e) {
log.error("[error report] 异常邮件发送失败, 重试中: {}", e.getMessage());
throw new ReportException();
}
}
@Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2))
public void reportToRemote(final ErrorReport report) {
try {
// TODO 2024/6/13: 发送到指定服务器上
// HttpClientUtils.sendPost();
log.info("[error report] 异常报告发送成功:{}", report.message);
} catch (Exception e) {
log.error("[error report] 邮件发送失败, 重试中: {}", e.getMessage());
throw new ReportException();
}
}
@Recover
public void recover(ReportException e) {
// recover 方法的返回值要和重试方法的返回值一致
log.error("[error report]:异常通知发送重试到达最大次数,发送失败");
}
public class ReportException extends RuntimeException {
}
}

@ -4,9 +4,11 @@ import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataImport;
import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.info.Constant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.DependsOn;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -19,6 +21,10 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
@ -36,7 +42,14 @@ public class GridDataProcessor {
GfsDownloader gfsDownloader;
@Resource
GfsDataConfig gfsDataConfig;
@Resource
ErrorReportService errorReportService;
@Value("${mail.send.to}")
private String[] mailReceiver;
private final ExecutorService executorService = Executors.newFixedThreadPool(3);
boolean test = false;
@Scheduled(cron = "0 0 0,8,12,18 * * ?")
public void dailyDataProcess() throws Exception {
if (download()) {
@ -45,48 +58,55 @@ public class GridDataProcessor {
clearExpiredData();
}
private boolean importToTableStore() throws Exception {
try {
OffsetDateTime refTime = gfsDownloader.getRefTime();
log.info("[schedule-start] 开始将数据导入数据库, 起报时间: {}", refTime);
List<GfsDataImport.ImportResult> importResults = gfsDataImport.importData(refTime);
List<GfsDataImport.ImportResult> failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule-end] 气象数据导入数据库失败, 文件列表: {} ", failedList);
emailReport();
return false;
}
log.info("[schedule-end] 气象数据导入数据库成功");
return true;
} catch (Exception e) {
log.error("[schedule-end] 气象数据导入数据库失败", e);
return false;
}
}
private boolean download() {
public boolean download() {
gfsDownloader.iniTimeSetting();
try {
log.info("[schedule-start] 开始下载气象数据");
gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
// if (test){
// throw new RuntimeException("测试异常");
// }
List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule] 下载失败文件列表: {}", failedList);
emailReport();
downloadFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
return false;
}
log.info("[schedule-end] 下载气象数据成功");
return true;
} catch (RuntimeException e) {
log.error("[schedule-end] 下载气象数据失败", e);
downloadFailedReport(gfsDownloader.getRefTimeStr(), e);
return false;
}
}
private void emailReport() {
public boolean importToTableStore(){
try {
OffsetDateTime refTime = gfsDownloader.getRefTime();
log.info("[schedule-start] 开始将数据导入数据库, 起报时间: {}", refTime);
// if (test){
// throw new RuntimeException("测试异常");
// }
List<ImportResult> importResults = gfsDataImport.importData(refTime);
List<ImportResult> finishedList = importResults.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
List<ImportResult> failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule-end] 气象数据导入数据库失败, 文件列表: {} ", failedList);
importFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
return false;
}
log.info("[schedule-end] 气象数据导入数据库成功");
return true;
} catch (Exception e) {
log.error("[schedule-end] 气象数据导入数据库失败", e);
importFailedReport(gfsDownloader.getRefTimeStr(),e);
return false;
}
}
public void clearExpiredData() {
log.info("[schedule-start] 开始清理过期数据");
File dataDir = new File(gfsDataConfig.getSaveRoot());
@ -108,4 +128,108 @@ public class GridDataProcessor {
}
}
private void downloadFailedReport(String refTime, List<FileInfo> finishedList, List<FileInfo> failedList) {
// 邮件
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
mail.setTos(mailReceiver);
StringBuilder sb = new StringBuilder();
sb.append("数据下载失败通知\n 气象数据下载失败,请检查下载配置。\n");
sb.append("起报时间: ").append(refTime).append("\n");
sb.append("下载失败文件列表: \n");
sb.append(failedList).append("\n");
sb.append("下载成功文件列表: \n");
sb.append(finishedList).append("\n");
mail.setContent(sb.toString());
Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
// post
ErrorReport errorReport = new ErrorReport();
errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime);
Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
try {
emailFuture.get();
httpFuture.get();
} catch (Exception ex) {
log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
}
}
private void downloadFailedReport(String refTime, Throwable e) {
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
mail.setTos(mailReceiver);
String sb = "数据下载失败通知\n 气象数据下载失败,请检查下载配置。\n" +
"起报时间: " + refTime + "。\n";
mail.setContent(sb + e);
Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
// post
ErrorReport errorReport = new ErrorReport();
errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime);
Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
try {
emailFuture.get();
httpFuture.get();
} catch (Exception ex) {
log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
}
}
private void importFailedReport(String refTime, List<ImportResult> finishedList, List<ImportResult> failedList) {
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据入库失败通知");
mail.setTos(mailReceiver);
StringBuilder sb = new StringBuilder();
sb.append("数据下载失败通知\n 数据入库下载失败,请检查。\n");
sb.append("起报时间: ").append(refTime).append("\n");
sb.append("失败文件列表: \n");
sb.append(failedList).append("\n");
sb.append("成功文件列表: \n");
sb.append(finishedList).append("\n");
mail.setContent(sb.toString());
Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
// post
ErrorReport errorReport = new ErrorReport();
errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime);
Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
try {
emailFuture.get();
httpFuture.get();
} catch (Exception ex) {
log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
}
}
private void importFailedReport(String refTime, Throwable e) {
Mail mail = new Mail();
mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
mail.setTos(mailReceiver);
String sb = "数据下载失败通知\n 气象数据下载失败,请检查下载配置。\n" +
"起报时间: " + refTime + "。\n";
mail.setContent(sb + e);
Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
// post
ErrorReport errorReport = new ErrorReport();
errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime);
Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
try {
emailFuture.get();
httpFuture.get();
} catch (Exception ex) {
log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
}
}
}

@ -0,0 +1,24 @@
package com.htfp.weather.schedule;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
/**
* @Author : shiyi
* @Date : 2024/6/13 14:08
* @Description :
*/
@Data
@ToString
public class Mail {
// 邮件主题
private String subject;
// 邮件内容
private String content;
// 收件邮箱列表
private String[] tos;
}

@ -9,4 +9,5 @@ tablestore:
endpoint: https://gfs-test.cn-hangzhou.ots.aliyuncs.com
accessId: LTAI5tDphvXLfwKJEmVQqrVz
accessKey: ksioVP1S36PI6plkIIRN4A2xLB94uc
instanceName: gfs-test
instanceName: gfs-test

@ -6,7 +6,18 @@ spring:
mvc:
servlet:
load-on-startup: 1 # 关闭 dispatcherServlet懒加载
thymeleaf:
cache: false
encoding: UTF-8
encoding: UTF-8
mail:
host: smtp.exmail.qq.com
username: shiyi@htsdfp.com
password: weatherHTFP123
properties.mail.smtp:
auth: true
enable: true
required: true
mail:
send:
from: shiyi@htsdfp.com
to: shiyi@htsdfp.com

@ -0,0 +1,29 @@
package com.htfp.weather.schedule;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Author : shiyi
* @Date : 2024/6/13 16:38
* @Description :
*/
@SpringBootTest
class GridDataProcessorTest {
@Resource
GridDataProcessor gridDataProcessor;
@Test
void importToTableStore() {
}
@Test
void download() {
gridDataProcessor.download();
}
}

@ -6,7 +6,18 @@ spring:
mvc:
servlet:
load-on-startup: 1 # 关闭 dispatcherServlet懒加载
thymeleaf:
cache: false
encoding: UTF-8
encoding: UTF-8
mail:
host: smtp.exmail.qq.com
username: shiyi@htsdfp.com
password: weatherHTFP123
properties.mail.smtp:
auth: true
enable: true
required: true
mail:
send:
from: shiyi@htsdfp.com
to: shiyi@htsdfp.com
Loading…
Cancel
Save