diff --git a/weather-service/pom.xml b/weather-service/pom.xml index a1f237d..33e9704 100644 --- a/weather-service/pom.xml +++ b/weather-service/pom.xml @@ -71,6 +71,18 @@ org.springframework.boot spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-mail + + + org.springframework.retry + spring-retry + + + org.aspectj + aspectjweaver + org.springframework.boot spring-boot-starter-test diff --git a/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java index 90797fe..14b9058 100644 --- a/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java +++ b/weather-service/src/main/java/com/htfp/weather/WeatherServiceApplication.java @@ -2,11 +2,13 @@ package com.htfp.weather; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.retry.annotation.EnableRetry; import org.springframework.scheduling.annotation.EnableScheduling; import java.util.TimeZone; -@EnableScheduling +@EnableRetry // 开启重试机制 +@EnableScheduling // 开启定时任务 @SpringBootApplication public class WeatherServiceApplication { diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java new file mode 100644 index 0000000..cbdb573 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReport.java @@ -0,0 +1,14 @@ +package com.htfp.weather.schedule; + +import lombok.Data; + +/** + * @Author : shiyi + * @Date : 2024/6/13 15:32 + * @Description : 异常报告内容 + */ +@Data +public class ErrorReport { + String time; + String message; +} diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java new file mode 100644 index 0000000..08f1212 --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/schedule/ErrorReportService.java @@ -0,0 +1,66 @@ +package com.htfp.weather.schedule; + +import com.htfp.weather.utils.HttpClientUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Recover; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; + +/** + * @Author : shiyi + * @Date : 2024/6/13 10:57 + * @Description : 邮件发送服务 + */ +@Service +@Slf4j +public class ErrorReportService { + @Autowired + private JavaMailSender mailSender; + + @Value("${mail.send.from}") + public String sender;// 发送者 + // 发送简单文本文件 + + @Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2)) + public void sendSimpleEmail(final Mail mail) { + try { + SimpleMailMessage message = new SimpleMailMessage(); + message.setFrom(sender); + message.setTo(mail.getTos()); + message.setSubject(mail.getSubject()); + message.setText(mail.getContent()); + mailSender.send(message); + log.info("[error report] 异常邮件发送成功,{} --> {}, 主题：{}", sender, mail.getTos(), mail.getSubject()); + } catch (Exception e) { + log.error("[error report] 异常邮件发送失败, 重试中： {}", e.getMessage()); + throw new ReportException(); + } + } + + @Retryable(value = {ReportException.class}, maxAttempts = 4, backoff = @Backoff(delay = 2000, multiplier = 2)) + public void reportToRemote(final ErrorReport report) { + try { + // TODO 2024/6/13: 发送到指定服务器上 + // HttpClientUtils.sendPost(); + log.info("[error report] 异常报告发送成功：{}", report.message); + } catch (Exception e) { + log.error("[error report] 邮件发送失败, 重试中： {}", e.getMessage()); + throw new ReportException(); + } + } + + @Recover + public void recover(ReportException e) { + // recover 方法的返回值要和重试方法的返回值一致 + log.error("[error report]：异常通知发送重试到达最大次数，发送失败"); + } + + public class ReportException extends RuntimeException { + } +} + diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java index 67d89dc..7494d52 100644 --- a/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java +++ b/weather-service/src/main/java/com/htfp/weather/schedule/GridDataProcessor.java @@ -4,9 +4,11 @@ import com.htfp.weather.download.FileInfo; import com.htfp.weather.download.GfsDataConfig; import com.htfp.weather.download.GfsDownloader; import com.htfp.weather.griddata.operation.GfsDataImport; +import com.htfp.weather.griddata.operation.GfsDataImport.ImportResult; import com.htfp.weather.info.Constant; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.DependsOn; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -19,6 +21,10 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.stream.Collectors; /** @@ -36,7 +42,14 @@ public class GridDataProcessor { GfsDownloader gfsDownloader; @Resource GfsDataConfig gfsDataConfig; + @Resource + ErrorReportService errorReportService; + + @Value("${mail.send.to}") + private String[] mailReceiver; + private final ExecutorService executorService = Executors.newFixedThreadPool(3); + boolean test = false; @Scheduled(cron = "0 0 0,8,12,18 * * ?") public void dailyDataProcess() throws Exception { if (download()) { @@ -45,48 +58,55 @@ public class GridDataProcessor { clearExpiredData(); } - private boolean importToTableStore() throws Exception { - try { - OffsetDateTime refTime = gfsDownloader.getRefTime(); - log.info("[schedule-start] 开始将数据导入数据库， 起报时间: {}", refTime); - List importResults = gfsDataImport.importData(refTime); - List failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList()); - if (!failedList.isEmpty()) { - log.error("[schedule-end] 气象数据导入数据库失败, 文件列表： {} ", failedList); - emailReport(); - return false; - } - log.info("[schedule-end] 气象数据导入数据库成功"); - return true; - } catch (Exception e) { - log.error("[schedule-end] 气象数据导入数据库失败", e); - return false; - } - } - - private boolean download() { + public boolean download() { + gfsDownloader.iniTimeSetting(); try { log.info("[schedule-start] 开始下载气象数据"); - gfsDownloader.iniTimeSetting(); List fileInfoList = gfsDownloader.getFilesInfo(); + // if (test){ + // throw new RuntimeException("测试异常"); + // } List finishedList = gfsDownloader.downloadAll(fileInfoList); List failedList = finishedList.stream().filter(fileInfo -> !fileInfo.isDownloadSuccess()).collect(Collectors.toList()); if (!failedList.isEmpty()) { log.error("[schedule] 下载失败文件列表： {}", failedList); - emailReport(); + downloadFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList); return false; } log.info("[schedule-end] 下载气象数据成功"); return true; } catch (RuntimeException e) { log.error("[schedule-end] 下载气象数据失败", e); + downloadFailedReport(gfsDownloader.getRefTimeStr(), e); return false; } } - private void emailReport() { + public boolean importToTableStore(){ + try { + OffsetDateTime refTime = gfsDownloader.getRefTime(); + log.info("[schedule-start] 开始将数据导入数据库， 起报时间: {}", refTime); + // if (test){ + // throw new RuntimeException("测试异常"); + // } + List importResults = gfsDataImport.importData(refTime); + List finishedList = importResults.stream().filter(result -> result.isSuccess()).collect(Collectors.toList()); + List failedList = importResults.stream().filter(result -> !result.isSuccess()).collect(Collectors.toList()); + if (!failedList.isEmpty()) { + log.error("[schedule-end] 气象数据导入数据库失败, 文件列表： {} ", failedList); + importFailedReport(gfsDownloader.getRefTimeStr(), finishedList, failedList); + return false; + } + log.info("[schedule-end] 气象数据导入数据库成功"); + return true; + } catch (Exception e) { + log.error("[schedule-end] 气象数据导入数据库失败", e); + importFailedReport(gfsDownloader.getRefTimeStr(),e); + return false; + } } + public void clearExpiredData() { log.info("[schedule-start] 开始清理过期数据"); File dataDir = new File(gfsDataConfig.getSaveRoot()); @@ -108,4 +128,108 @@ public class GridDataProcessor { } } + private void downloadFailedReport(String refTime, List finishedList, List failedList) { + // 邮件 + Mail mail = new Mail(); + mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知"); + mail.setTos(mailReceiver); + StringBuilder sb = new StringBuilder(); + sb.append("数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n"); + sb.append("起报时间: ").append(refTime).append("\n"); + sb.append("下载失败文件列表: \n"); + sb.append(failedList).append("\n"); + sb.append("下载成功文件列表: \n"); + sb.append(finishedList).append("\n"); + mail.setContent(sb.toString()); + Future emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail)); + + // post + ErrorReport errorReport = new ErrorReport(); + errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); + errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime); + Future httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport)); + + try { + emailFuture.get(); + httpFuture.get(); + } catch (Exception ex) { + log.error("[schedule] 异常报告过程错误: {}", ex.getMessage()); + } + } + + private void downloadFailedReport(String refTime, Throwable e) { + Mail mail = new Mail(); + mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知"); + mail.setTos(mailReceiver); + String sb = "数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n" + + "起报时间: " + refTime + "。\n"; + + mail.setContent(sb + e); + Future emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail)); + + // post + ErrorReport errorReport = new ErrorReport(); + errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); + errorReport.setMessage("【航天飞鹏-气象服务】数据下载失败通知, 起报时间:" + refTime); + Future httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport)); + + try { + emailFuture.get(); + httpFuture.get(); + } catch (Exception ex) { + log.error("[schedule] 异常报告过程错误: {}", ex.getMessage()); + } + } + + private void importFailedReport(String refTime, List finishedList, List failedList) { + Mail mail = new Mail(); + mail.setSubject("【航天飞鹏-气象服务】数据入库失败通知"); + mail.setTos(mailReceiver); + StringBuilder sb = new StringBuilder(); + sb.append("数据下载失败通知\n 数据入库下载失败，请检查。\n"); + sb.append("起报时间: ").append(refTime).append("\n"); + sb.append("失败文件列表: \n"); + sb.append(failedList).append("\n"); + sb.append("成功文件列表: \n"); + sb.append(finishedList).append("\n"); + mail.setContent(sb.toString()); + Future emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail)); + + // post + ErrorReport errorReport = new ErrorReport(); + errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); + errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime); + Future httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport)); + + try { + emailFuture.get(); + httpFuture.get(); + } catch (Exception ex) { + log.error("[schedule] 异常报告过程错误: {}", ex.getMessage()); + } + } + + private void importFailedReport(String refTime, Throwable e) { + Mail mail = new Mail(); + mail.setSubject("【航天飞鹏-气象服务】数据下载失败通知"); + mail.setTos(mailReceiver); + String sb = "数据下载失败通知\n 气象数据下载失败，请检查下载配置。\n" + + "起报时间: " + refTime + "。\n"; + mail.setContent(sb + e); + Future emailFuture = executorService.submit(() -> errorReportService.sendSimpleEmail(mail)); + + // post + ErrorReport errorReport = new ErrorReport(); + errorReport.setTime(OffsetDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); + errorReport.setMessage("【航天飞鹏-气象服务】数据入库失败通知, 起报时间:" + refTime); + Future httpFuture = executorService.submit(() -> errorReportService.reportToRemote(errorReport)); + + try { + emailFuture.get(); + httpFuture.get(); + } catch (Exception ex) { + log.error("[schedule] 异常报告过程错误: {}", ex.getMessage()); + } + } + } diff --git a/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java b/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java new file mode 100644 index 0000000..6b5a17c --- /dev/null +++ b/weather-service/src/main/java/com/htfp/weather/schedule/Mail.java @@ -0,0 +1,24 @@ +package com.htfp.weather.schedule; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.io.Serializable; + +/** + * @Author : shiyi + * @Date : 2024/6/13 14:08 + * @Description : 邮件 + */ +@Data +@ToString +public class Mail { + // 邮件主题 + private String subject; + // 邮件内容 + private String content; + // 收件邮箱列表 + private String[] tos; +} diff --git a/weather-service/src/main/resources/application-weather.yml b/weather-service/src/main/resources/application-weather.yml index d73b258..e409048 100644 --- a/weather-service/src/main/resources/application-weather.yml +++ b/weather-service/src/main/resources/application-weather.yml @@ -9,4 +9,5 @@ tablestore: endpoint: https://gfs-test.cn-hangzhou.ots.aliyuncs.com accessId: LTAI5tDphvXLfwKJEmVQqrVz accessKey: ksioVP1S36PI6plkIIRN4A2xLB94uc - instanceName: gfs-test \ No newline at end of file + instanceName: gfs-test + diff --git a/weather-service/src/main/resources/application.yml b/weather-service/src/main/resources/application.yml index cd4c264..727fc6d 100644 --- a/weather-service/src/main/resources/application.yml +++ b/weather-service/src/main/resources/application.yml @@ -6,7 +6,18 @@ spring: mvc: servlet: load-on-startup: 1 # 关闭 dispatcherServlet懒加载 - thymeleaf: cache: false - encoding: UTF-8 \ No newline at end of file + encoding: UTF-8 + mail: + host: smtp.exmail.qq.com + username: shiyi@htsdfp.com + password: weatherHTFP123 + properties.mail.smtp: + auth: true + enable: true + required: true +mail: + send: + from: shiyi@htsdfp.com + to: shiyi@htsdfp.com \ No newline at end of file diff --git a/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java new file mode 100644 index 0000000..51d438c --- /dev/null +++ b/weather-service/src/test/java/com/htfp/weather/schedule/GridDataProcessorTest.java @@ -0,0 +1,29 @@ +package com.htfp.weather.schedule; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @Author : shiyi + * @Date : 2024/6/13 16:38 + * @Description : + */ +@SpringBootTest +class GridDataProcessorTest { + + @Resource + GridDataProcessor gridDataProcessor; + @Test + void importToTableStore() { + + } + + @Test + void download() { + gridDataProcessor.download(); + } +} \ No newline at end of file diff --git a/weather-service/src/test/resources/application.yml b/weather-service/src/test/resources/application.yml index cd4c264..727fc6d 100644 --- a/weather-service/src/test/resources/application.yml +++ b/weather-service/src/test/resources/application.yml @@ -6,7 +6,18 @@ spring: mvc: servlet: load-on-startup: 1 # 关闭 dispatcherServlet懒加载 - thymeleaf: cache: false - encoding: UTF-8 \ No newline at end of file + encoding: UTF-8 + mail: + host: smtp.exmail.qq.com + username: shiyi@htsdfp.com + password: weatherHTFP123 + properties.mail.smtp: + auth: true + enable: true + required: true +mail: + send: + from: shiyi@htsdfp.com + to: shiyi@htsdfp.com \ No newline at end of file