third / fw-freya

Commit 580a4f0e972ec7a210c6f3c432e3918dc9d856bb

Authored by 廖洋铭
1 parent e7407fd4

短视频调整为采集截止昨日前发布视频

Inline Side-by-side
Showing 5 changed files with 18 additions and 5 deletions
src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java
  View file @580a4f0
... ... @@ -85,7 +85,7 @@ public interface CrawlStrategy {
85 85 }
86 86  
87 87 /**
88   - * 获取所有视频信息
  88 + * 获取截止昨日发布所有视频信息
89 89 *
90 90 * @param accountNo 账户号
91 91 * @return
... ...
src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java
  View file @580a4f0
... ... @@ -207,7 +207,10 @@ public class BilibiliCrawl implements CrawlStrategy {
207 207 });
208 208 try {
209 209 if (!videoPoolList.isEmpty()) {
210   - videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
  210 + Date endTime = DateUtil.getThisDayMaxTime(previousDay);
  211 + videoPoolDao.saveAll(videoPoolList.stream()
  212 + .filter(r -> r.getPublishTime().compareTo(endTime) <= 0)
  213 + .collect(Collectors.toList()));// 将收集到的视频信息保存
211 214 } else {
212 215 final VideoPool nullVideo = VideoPool.builder()
213 216 .accountNo(accountNo)
... ...
src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java
  View file @580a4f0
... ... @@ -232,7 +232,10 @@ public class DongCheDiCrawl implements CrawlStrategy {
232 232 });
233 233 try {
234 234 if (!videoPoolList.isEmpty()) {
235   - videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
  235 + Date endTime = DateUtil.getThisDayMaxTime(previousDay);
  236 + videoPoolDao.saveAll(videoPoolList.stream()
  237 + .filter(r -> r.getPublishTime().compareTo(endTime) <= 0)
  238 + .collect(Collectors.toList()));// 将收集到的视频信息保存
236 239 } else {
237 240 final VideoPool nullVideo = VideoPool.builder()
238 241 .accountNo(accountNo)
... ...
src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java
  View file @580a4f0
... ... @@ -390,7 +390,11 @@ public class DouYinCrawl implements CrawlStrategy {
390 390 }
391 391 try {
392 392 if (!videoPoolList.isEmpty()) {
393   - videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
  393 + Date previousDay = DateUtil.getPreviousDay(new Date());
  394 + Date endTime = DateUtil.getThisDayMaxTime(previousDay);
  395 + videoPoolDao.saveAll(videoPoolList.stream()
  396 + .filter(r -> r.getPublishTime().compareTo(endTime) <= 0)
  397 + .collect(Collectors.toList()));// 将收集到的视频信息保存
394 398 } else {
395 399 videoPoolDao.save(VideoPool.builder()
396 400 .accountNo(accountNo)
... ...
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @580a4f0
... ... @@ -434,7 +434,10 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
434 434 });
435 435 try {
436 436 if (!videoPoolList.isEmpty()) {
437   - videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
  437 + Date endTime = DateUtil.getThisDayMaxTime(previousDay);
  438 + videoPoolDao.saveAll(videoPoolList.stream()
  439 + .filter(r -> r.getPublishTime().compareTo(endTime) <= 0)
  440 + .collect(Collectors.toList()));// 将收集到的视频信息保存
438 441 } else {
439 442 final VideoPool nullVideo = VideoPool.builder()
440 443 .accountNo(accountNo)
... ...