third / fw-freya

Commit b1586371ceacb43eac09d4f73f8af61281314812

Authored by 王明元
1 parent b8357e80

2022年6月1日15:06:21 调整线程池队列长度, 修改回播匹配规则提高回播匹配率, 加快回放抓取频率

Inline Side-by-side
Showing 3 changed files with 6 additions and 4 deletions
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @b158637
... ... @@ -466,7 +466,9 @@ public class KuaiShouCrawl implements CrawlStrategy {
466 466 final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
467 467 // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等
468 468 // (liveStartSub < 2)->说明开播时间几乎一样
469   - if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) {
  469 + if (!((Objects.equals(liveDuration, duration) ||
  470 + (timeSubAbs < 5)) && liveStartSub < 10) ||
  471 + !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 1000 && duration >= durationThreshold)) {
470 472 playbackMsg = null;
471 473 log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
472 474 }
... ...
src/main/java/cn/fw/freya/task/DataCaptureTask.java
  View file @b158637
... ... @@ -112,7 +112,7 @@ public class DataCaptureTask {
112 112 .stream()
113 113 .peek(item -> item.sort(Comparator.comparing(LivePool::getGetPlaybackFailTimes)))// 把某个人每条直播按失败次数排序
114 114 .sorted(Comparator.comparing(item -> item.get(0).getGetPlaybackFailTimes()))// 失败次数最小的人排到前面
115   - .limit(3)// 一次找3个人的直播
  115 + .limit(5)// 一次找5个人的直播
116 116 .collect(Collectors.toList());
117 117 for (List<LivePool> list : values) {// 遍历每个人的直播集合
118 118 final String accountNo = list.get(0).getPhoneNo();
... ... @@ -151,7 +151,7 @@ public class DataCaptureTask {
151 151 livePoolDao.saveAll(livePools);
152 152 publisher.publishEvent(account);// 发布事件
153 153 }
154   - LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(20 + random.nextInt(15)));
  154 + LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(12 + random.nextInt(8)));
155 155 }
156 156 }
157 157  
... ...
src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java
  View file @b158637
... ... @@ -33,7 +33,7 @@ public class ThreadPoolUtil {
33 33 12,
34 34 60,
35 35 TimeUnit.SECONDS,
36   - new LinkedBlockingQueue<>(256),
  36 + new LinkedBlockingQueue<>(128),
37 37 new ThreadFactoryBuilder().setNamePrefix("capture-pool-").build(),
38 38 new ThreadPoolExecutor.DiscardPolicy());// 自行创建线程池, 并将创建好的线程池对象赋值给类的静态成员变量threadPool
39 39 }
... ...