Commit f9c3021644b1c1faca91b5d1c5a364f9fa9d59bd
1 parent
36784d5a
2022年5月26日16:21:27 保存直播数据时初始化失败次数为0
Showing
5 changed files
with
6 additions
and
2 deletions
src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java
@@ -379,6 +379,7 @@ public class BilibiliCrawl implements CrawlStrategy { | @@ -379,6 +379,7 @@ public class BilibiliCrawl implements CrawlStrategy { | ||
379 | //.roomCoverImage(Optional.ofNullable(obj.getString("liveCover")).orElse("")) | 379 | //.roomCoverImage(Optional.ofNullable(obj.getString("liveCover")).orElse("")) |
380 | .roomId(Optional.ofNullable(obj.getString("live_id")).orElse(""))// 直播间id | 380 | .roomId(Optional.ofNullable(obj.getString("live_id")).orElse(""))// 直播间id |
381 | .roomName(Optional.ofNullable(obj.getString("title")).orElse(""))// 直播间标题 | 381 | .roomName(Optional.ofNullable(obj.getString("title")).orElse(""))// 直播间标题 |
382 | + .getPlaybackFailTimes(0) | ||
382 | //.score(obj.getInteger("score")) | 383 | //.score(obj.getInteger("score")) |
383 | //.watchCnt(obj.getInteger("watch_cnt")) | 384 | //.watchCnt(obj.getInteger("watch_cnt")) |
384 | //.watchPeakUserCnt(Optional.ofNullable(obj.getInteger("max_online")).orElse(0))// 在线峰值??? | 385 | //.watchPeakUserCnt(Optional.ofNullable(obj.getInteger("max_online")).orElse(0))// 在线峰值??? |
src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java
@@ -331,6 +331,7 @@ public class DongCheDiCrawl implements CrawlStrategy { | @@ -331,6 +331,7 @@ public class DongCheDiCrawl implements CrawlStrategy { | ||
331 | //.shareCnt(Optional.ofNullable(obj.getInteger("shareUv")).orElse(0)) | 331 | //.shareCnt(Optional.ofNullable(obj.getInteger("shareUv")).orElse(0)) |
332 | //.likeCnt(Optional.ofNullable(obj.getInteger("likeCnt")).orElse(0)) | 332 | //.likeCnt(Optional.ofNullable(obj.getInteger("likeCnt")).orElse(0)) |
333 | .userNick(Optional.ofNullable(userInfo.getString("userNick")).orElse(""))// 用户昵称 | 333 | .userNick(Optional.ofNullable(userInfo.getString("userNick")).orElse(""))// 用户昵称 |
334 | + .getPlaybackFailTimes(0) | ||
334 | .build()); | 335 | .build()); |
335 | }); | 336 | }); |
336 | Date previousDay = DateUtil.getPreviousDay(new Date()); | 337 | Date previousDay = DateUtil.getPreviousDay(new Date()); |
src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java
@@ -582,6 +582,7 @@ public class DouYinCrawl implements CrawlStrategy { | @@ -582,6 +582,7 @@ public class DouYinCrawl implements CrawlStrategy { | ||
582 | .watchCnt(Optional.ofNullable(obj.getInteger("watch_cnt")).orElse(0))// 观看人次 | 582 | .watchCnt(Optional.ofNullable(obj.getInteger("watch_cnt")).orElse(0))// 观看人次 |
583 | .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("watch_pcu")).orElse(0))// 在线人数巅峰 | 583 | .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("watch_pcu")).orElse(0))// 在线人数巅峰 |
584 | .watchUserCnt(Optional.ofNullable(obj.getInteger("watch_ucnt")).orElse(0))// 观众人数 | 584 | .watchUserCnt(Optional.ofNullable(obj.getInteger("watch_ucnt")).orElse(0))// 观众人数 |
585 | + .getPlaybackFailTimes(0) | ||
585 | .build() | 586 | .build() |
586 | ); | 587 | ); |
587 | } | 588 | } |
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
@@ -419,6 +419,7 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -419,6 +419,7 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
419 | //.watchCnt(obj.getInteger("watch_cnt")) | 419 | //.watchCnt(obj.getInteger("watch_cnt")) |
420 | .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("maxConcurrentUv")).orElse(0)) | 420 | .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("maxConcurrentUv")).orElse(0)) |
421 | .watchUserCnt(Optional.ofNullable(obj.getInteger("playUv")).orElse(0)) | 421 | .watchUserCnt(Optional.ofNullable(obj.getInteger("playUv")).orElse(0)) |
422 | + .getPlaybackFailTimes(0) | ||
422 | 423 | ||
423 | .shareCnt(Optional.ofNullable(obj.getInteger("shareUv")).orElse(0)) | 424 | .shareCnt(Optional.ofNullable(obj.getInteger("shareUv")).orElse(0)) |
424 | .likeCnt(Optional.ofNullable(obj.getInteger("likeCnt")).orElse(0)) | 425 | .likeCnt(Optional.ofNullable(obj.getInteger("likeCnt")).orElse(0)) |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
@@ -101,8 +101,8 @@ public class DataCaptureTask { | @@ -101,8 +101,8 @@ public class DataCaptureTask { | ||
101 | final Random random = new Random(); | 101 | final Random random = new Random(); |
102 | List<LivePool> withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) | 102 | List<LivePool> withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) |
103 | .stream() | 103 | .stream() |
104 | - .filter(item -> 0 < item.getGetPlaybackFailTimes() && item.getGetPlaybackFailTimes() < 15) | ||
105 | - .collect(Collectors.toList()); | 104 | + .filter(item -> 0 <= item.getGetPlaybackFailTimes() && item.getGetPlaybackFailTimes() < 30) |
105 | + .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据 | ||
106 | Collection<List<LivePool>> values = withoutPlaybackLive | 106 | Collection<List<LivePool>> values = withoutPlaybackLive |
107 | .stream() | 107 | .stream() |
108 | .collect(Collectors.groupingBy(LivePool::getPhoneNo)) | 108 | .collect(Collectors.groupingBy(LivePool::getPhoneNo)) |