diff --git a/weather-service/pom.xml b/weather-service/pom.xml
index a1f237d..33e9704 100644
--- a/weather-service/pom.xml
+++ b/weather-service/pom.xml
@@ -71,6 +71,18 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-mail</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.retry</groupId>
+ <artifactId>spring-retry</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjweaver</artifactId>
+ </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
diff --git a/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java
index 90797fe..14b9058 100644
--- a/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java
+++ b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java
@@ -2,11 +2,13 @@ package com.htfp.weather;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.retry.annotation.EnableRetry;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.util.TimeZone;
-@EnableScheduling
+@EnableRetry // 开启重试机制
+@EnableScheduling // 开启定时任务
@SpringBootApplication
public class WeatherServiceApplication {
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java
new file mode 100644
index 0000000..cbdb573
--- /dev/null
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java
@@ -0,0 +1,14 @@
+package com.htfp.weather.schedule;
+
+import lombok.Data;
+
+/**
+ * @Author : shiyi
+ * @Date : 2024/6/13 15:32
+ * @Description : 异常报告内容
+ */
+@Data
+public class ErrorReport {
+ String time;
+ String message;
+}
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java
new file mode 100644
index 0000000..08f1212
--- /dev/null
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java
@@ -0,0 +1,66 @@
+package com.htfp.weather.schedule;
+
+import com.htfp.weather.utils.HttpClientUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.mail.SimpleMailMessage;
+import org.springframework.mail.javamail.JavaMailSender;
+import org.springframework.retry.annotation.Backoff;
+import org.springframework.retry.annotation.Recover;
+import org.springframework.retry.annotation.Retryable;
+import org.springframework.stereotype.Service;
+
+/**
+ * @Author : shiyi
+ * @Date : 2024/6/13 10:57
+ * @Description : 邮件发送服务
+ */
+@Service
+@Slf4j
+public class ErrorReportService {
+ @Autowired
+ private JavaMailSender mailSender;
+
+ @Value("${mail.send.from}")
+ public String sender;// 发送者
+ // 发送简单文本文件
+
+ @Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2))
+ public void sendSimpleEmail(final Mail mail) {
+ try {
+ SimpleMailMessage message = new SimpleMailMessage();
+ message.setFrom(sender);
+ message.setTo(mail.getTos());
+ message.setSubject(mail.getSubject());
+ message.setText(mail.getContent());
+ mailSender.send(message);
+ log.info("[error report] 异常邮件发送成功,{} --> {}, 主题：{}", sender, mail.getTos(), mail.getSubject());
+ } catch (Exception e) {
+ log.error("[error report] 异常邮件发送失败, 重试中： {}", e.getMessage());
+ throw new ReportException();
+ }
+ }
+
+ @Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2))
+ public void reportToRemote(final ErrorReport report) {
+ try {
+ // TODO 2024/6/13: 发送到指定服务器上
+ // HttpClientUtils.sendPost();
+ log.info("[error report] 异常报告发送成功：{}", report.message);
+ } catch (Exception e) {
+ log.error("[error report] 邮件发送失败, 重试中： {}", e.getMessage());
+ throw new ReportException();
+ }
+ }
+
+ @Recover
+ public void recover(ReportException e) {
+ // recover 方法的返回值要和重试方法的返回值一致
+ log.error("[error report]：异常通知发送重试到达最大次数，发送失败");
+ }
+
+ public class ReportException extends RuntimeException {
+ }
+}
+
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
index 67d89dc..7494d52 100644
--- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java
@@ -4,9 +4,11 @@ import com.htfp.weather.download.FileInfo;
import com.htfp.weather.download.GfsDataConfig;
import com.htfp.weather.download.GfsDownloader;
import com.htfp.weather.griddata.operation.GfsDataImport;
+import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult;
import com.htfp.weather.info.Constant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.DependsOn;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@@ -19,6 +21,10 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
@@ -36,7 +42,14 @@ public class GridDataProcessor {
GfsDownloader gfsDownloader;
@Resource
GfsDataConfig gfsDataConfig;
+ @Resource
+ ErrorReportService errorReportService;
+
+ @Value("${mail.send.to}")
+ private String[] mailReceiver;
+ private final ExecutorService executorService = Executors.newFixedThreadPool(3);
+ boolean test = false;
@Scheduled(cron = "0 0 0,8,12,18 * * ?")
public void dailyDataProcess() throws Exception {
if (download()) {
@@ -45,48 +58,55 @@ public class GridDataProcessor {
clearExpiredData();
}
- private boolean importToTableStore() throws Exception {
- try {
- OffsetDateTime refTime = gfsDownloader.getRefTime();
- log.info("[schedule-start] 开始将数据导入数据库， 起报时间: {}", refTime);
- List<GfsDataImport.ImportResult> importResults = gfsDataImport.importData(refTime);
- List<GfsDataImport.ImportResult> failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
- if (!failedList.isEmpty()) {
- log.error("[schedule-end] 气象数据导入数据库失败, 文件列表： {} ", failedList);
- emailReport();
- return false;
- }
- log.info("[schedule-end] 气象数据导入数据库成功");
- return true;
- } catch (Exception e) {
- log.error("[schedule-end] 气象数据导入数据库失败", e);
- return false;
- }
- }
-
- private boolean download() {
+ public boolean download() {
+ gfsDownloader.iniTimeSetting();
try {
log.info("[schedule-start] 开始下载气象数据");
- gfsDownloader.iniTimeSetting();
List<FileInfo> fileInfoList = gfsDownloader.getFilesInfo();
+ // if (test){
+ // throw new RuntimeException("测试异常");
+ // }
List<FileInfo> finishedList = gfsDownloader.downloadAll(fileInfoList);
List<FileInfo> failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList());
if (!failedList.isEmpty()) {
log.error("[schedule] 下载失败文件列表： {}", failedList);
- emailReport();
+ downloadFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
return false;
}
log.info("[schedule-end] 下载气象数据成功");
return true;
} catch (RuntimeException e) {
log.error("[schedule-end] 下载气象数据失败", e);
+ downloadFailedReport(gfsDownloader.getRefTimeStr(), e);
return false;
}
}
- private void emailReport() {
+ public boolean importToTableStore(){
+ try {
+ OffsetDateTime refTime = gfsDownloader.getRefTime();
+ log.info("[schedule-start] 开始将数据导入数据库， 起报时间: {}", refTime);
+ // if (test){
+ // throw new RuntimeException("测试异常");
+ // }
+ List<ImportResult> importResults = gfsDataImport.importData(refTime);
+ List<ImportResult> finishedList = importResults.stream().filter(result -> result.isSuccess()).collect(Collectors.toList());
+ List<ImportResult> failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList());
+ if (!failedList.isEmpty()) {
+ log.error("[schedule-end] 气象数据导入数据库失败, 文件列表： {} ", failedList);
+ importFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList);
+ return false;
+ }
+ log.info("[schedule-end] 气象数据导入数据库成功");
+ return true;
+ } catch (Exception e) {
+ log.error("[schedule-end] 气象数据导入数据库失败", e);
+ importFailedReport(gfsDownloader.getRefTimeStr(),e);
+ return false;
+ }
}
+
public void clearExpiredData() {
log.info("[schedule-start] 开始清理过期数据");
File dataDir = new File(gfsDataConfig.getSaveRoot());
@@ -108,4 +128,108 @@ public class GridDataProcessor {
}
}
+ private void downloadFailedReport(String refTime, List<FileInfo> finishedList, List<FileInfo> failedList) {
+ // 邮件
+ Mail mail = new Mail();
+ mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
+ mail.setTos(mailReceiver);
+ StringBuilder sb = new StringBuilder();
+ sb.append("数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n");
+ sb.append("起报时间: ").append(refTime).append("\n");
+ sb.append("下载失败文件列表: \n");
+ sb.append(failedList).append("\n");
+ sb.append("下载成功文件列表: \n");
+ sb.append(finishedList).append("\n");
+ mail.setContent(sb.toString());
+ Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
+
+ // post
+ ErrorReport errorReport = new ErrorReport();
+ errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime);
+ Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
+
+ try {
+ emailFuture.get();
+ httpFuture.get();
+ } catch (Exception ex) {
+ log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
+ }
+ }
+
+ private void downloadFailedReport(String refTime, Throwable e) {
+ Mail mail = new Mail();
+ mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
+ mail.setTos(mailReceiver);
+ String sb = "数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n" +
+ "起报时间: " + refTime + "。\n";
+
+ mail.setContent(sb + e);
+ Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
+
+ // post
+ ErrorReport errorReport = new ErrorReport();
+ errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime);
+ Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
+
+ try {
+ emailFuture.get();
+ httpFuture.get();
+ } catch (Exception ex) {
+ log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
+ }
+ }
+
+ private void importFailedReport(String refTime, List<ImportResult> finishedList, List<ImportResult> failedList) {
+ Mail mail = new Mail();
+ mail.setSubject("【航天飞鹏-气象服务】数据入库失败通知");
+ mail.setTos(mailReceiver);
+ StringBuilder sb = new StringBuilder();
+ sb.append("数据下载失败通知\n 数据入库下载失败，请检查。\n");
+ sb.append("起报时间: ").append(refTime).append("\n");
+ sb.append("失败文件列表: \n");
+ sb.append(failedList).append("\n");
+ sb.append("成功文件列表: \n");
+ sb.append(finishedList).append("\n");
+ mail.setContent(sb.toString());
+ Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
+
+ // post
+ ErrorReport errorReport = new ErrorReport();
+ errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime);
+ Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
+
+ try {
+ emailFuture.get();
+ httpFuture.get();
+ } catch (Exception ex) {
+ log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
+ }
+ }
+
+ private void importFailedReport(String refTime, Throwable e) {
+ Mail mail = new Mail();
+ mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知");
+ mail.setTos(mailReceiver);
+ String sb = "数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n" +
+ "起报时间: " + refTime + "。\n";
+ mail.setContent(sb + e);
+ Future<?> emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail));
+
+ // post
+ ErrorReport errorReport = new ErrorReport();
+ errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime);
+ Future<?> httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport));
+
+ try {
+ emailFuture.get();
+ httpFuture.get();
+ } catch (Exception ex) {
+ log.error("[schedule] 异常报告过程错误: {}", ex.getMessage());
+ }
+ }
+
}
diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java b/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java
new file mode 100644
index 0000000..6b5a17c
--- /dev/null
+++ b/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java
@@ -0,0 +1,24 @@
+package com.htfp.weather.schedule;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+import java.io.Serializable;
+
+/**
+ * @Author : shiyi
+ * @Date : 2024/6/13 14:08
+ * @Description : 邮件
+ */
+@Data
+@ToString
+public class Mail {
+ // 邮件主题
+ private String subject;
+ // 邮件内容
+ private String content;
+ // 收件邮箱列表
+ private String[] tos;
+}
diff --git a/weather-service/src/main/resources/application-weather.yml b/weather-service/src/main/resources/application-weather.yml
index d73b258..e409048 100644
--- a/weather-service/src/main/resources/application-weather.yml
+++ b/weather-service/src/main/resources/application-weather.yml
@@ -9,4 +9,5 @@ tablestore:
endpoint: https://gfs-test.cn-hangzhou.ots.aliyuncs.com
accessId: LTAI5tDphvXLfwKJEmVQqrVz
accessKey: ksioVP1S36PI6plkIIRN4A2xLB94uc
- instanceName: gfs-test
\ No newline at end of file
+ instanceName: gfs-test
+
diff --git a/weather-service/src/main/resources/application.yml b/weather-service/src/main/resources/application.yml
index cd4c264..727fc6d 100644
--- a/weather-service/src/main/resources/application.yml
+++ b/weather-service/src/main/resources/application.yml
@@ -6,7 +6,18 @@ spring:
mvc:
servlet:
load-on-startup: 1 # 关闭 dispatcherServlet懒加载
-
thymeleaf:
cache: false
- encoding: UTF-8
\ No newline at end of file
+ encoding: UTF-8
+ mail:
+ host: smtp.exmail.qq.com
+ username: shiyi@htsdfp.com
+ password: weatherHTFP123
+ properties.mail.smtp:
+ auth: true
+ enable: true
+ required: true
+mail:
+ send:
+ from: shiyi@htsdfp.com
+ to: shiyi@htsdfp.com
\ No newline at end of file
diff --git a/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java
new file mode 100644
index 0000000..51d438c
--- /dev/null
+++ b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java
@@ -0,0 +1,29 @@
+package com.htfp.weather.schedule;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import javax.annotation.Resource;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * @Author : shiyi
+ * @Date : 2024/6/13 16:38
+ * @Description :
+ */
+@SpringBootTest
+class GridDataProcessorTest {
+
+ @Resource
+ GridDataProcessor gridDataProcessor;
+ @Test
+ void importToTableStore() {
+
+ }
+
+ @Test
+ void download() {
+ gridDataProcessor.download();
+ }
+}
\ No newline at end of file
diff --git a/weather-service/src/test/resources/application.yml b/weather-service/src/test/resources/application.yml
index cd4c264..727fc6d 100644
--- a/weather-service/src/test/resources/application.yml
+++ b/weather-service/src/test/resources/application.yml
@@ -6,7 +6,18 @@ spring:
mvc:
servlet:
load-on-startup: 1 # 关闭 dispatcherServlet懒加载
-
thymeleaf:
cache: false
- encoding: UTF-8
\ No newline at end of file
+ encoding: UTF-8
+ mail:
+ host: smtp.exmail.qq.com
+ username: shiyi@htsdfp.com
+ password: weatherHTFP123
+ properties.mail.smtp:
+ auth: true
+ enable: true
+ required: true
+mail:
+ send:
+ from: shiyi@htsdfp.com
+ to: shiyi@htsdfp.com
\ No newline at end of file