third / fw-freya

Commit 1a95281dd5b9d08fa191fd5a00b5fef2808d5542

Authored by 王明元
1 parent 5736d4ca

2023年1月9日15:47:56 修复快手回播无法获取问题

Inline Side-by-side
Showing 2 changed files with 15 additions and 20 deletions
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @1a95281
... ... @@ -52,7 +52,6 @@ import java.io.FileOutputStream;
52 52 import java.io.IOException;
53 53 import java.io.OutputStreamWriter;
54 54 import java.math.BigDecimal;
55   -import java.math.RoundingMode;
56 55 import java.net.URLDecoder;
57 56 import java.nio.charset.StandardCharsets;
58 57 import java.time.LocalDate;
... ... @@ -612,6 +611,7 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
612 611 }
613 612  
614 613 public JSONObject setPlaybackUrl(LivePool dbLive, List<JSONObject> collect, Double durationThreshold) {
  614 + final int maxSubRangeMinutes = 2;// 直播结束时间与回播创建时间允许最大偏差分钟数
615 615 JSONObject object = new JSONObject();
616 616 object.put("coverUrl", null);
617 617 object.put("playbackUrl", null);
... ... @@ -624,25 +624,26 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
624 624 JSONObject playbackMsg;
625 625 if (Objects.equals(collect.size(), 1)) {
626 626 playbackMsg = collect.get(0);
627   - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长
  627 + /*Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长
628 628 final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳
629 629 final double timeSubAbs = Math.abs(liveDuration - duration);
630   - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
  630 + final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();*/
631 631 // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等
632 632 // (liveStartSub < 2)->说明开播时间几乎一样
633   - if ((!((Objects.equals(liveDuration, duration) ||
  633 + /*if ((!((Objects.equals(liveDuration, duration) ||
634 634 (timeSubAbs < 5)) && liveStartSub < 10)) &&
635   - !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold)) {
  635 + !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold))*/
  636 + if (Math.abs(playbackMsg.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) > maxSubRangeMinutes * 60 * 1000) {
636 637 playbackMsg = null;
637 638 log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
638 639 }
639 640 } else {
640   - List<JSONObject> collect1 = collect.stream().filter(item1 -> {
641   - Double duration = item1.getDouble("duration");
642   - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳
643   - final double timeSubAbs = Math.abs(liveDuration - duration);
644   - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
645   - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10;
  641 + List<JSONObject> collect1 = collect.stream().filter(item -> {
  642 + //Double duration = item.getDouble("duration");
  643 + //final Long startTimeStamp = item.getLong("startTime");// 回放信息返回直播开始时间戳
  644 + //final double timeSubAbs = Math.abs(liveDuration - duration);
  645 + //final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
  646 + return Math.abs(item.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) < maxSubRangeMinutes * 60 * 1000;
646 647 }).collect(Collectors.toList());
647 648 if (Objects.equals(collect1.size(), 1)) {
648 649 playbackMsg = collect1.get(0);
... ...
src/main/java/cn/fw/freya/task/DataCaptureTask.java
  View file @1a95281
... ... @@ -11,13 +11,8 @@ import cn.fw.freya.service.data.AccountService;
11 11 import cn.fw.freya.utils.DateUtil;
12 12 import cn.fw.freya.utils.PublicUtil;
13 13 import cn.fw.freya.utils.ThreadPoolUtil;
14   -import com.alibaba.fastjson.JSON;
15 14 import com.alibaba.fastjson.JSONArray;
16 15 import com.alibaba.fastjson.JSONObject;
17   -import com.alibaba.fastjson.annotation.JSONField;
18   -import lombok.AllArgsConstructor;
19   -import lombok.Data;
20   -import lombok.NoArgsConstructor;
21 16 import lombok.RequiredArgsConstructor;
22 17 import lombok.extern.slf4j.Slf4j;
23 18 import org.springframework.context.ApplicationEventPublisher;
... ... @@ -26,8 +21,6 @@ import org.springframework.stereotype.Component;
26 21 import org.springframework.util.CollectionUtils;
27 22  
28 23 import java.io.IOException;
29   -import java.math.BigDecimal;
30   -import java.math.RoundingMode;
31 24 import java.util.*;
32 25 import java.util.concurrent.BlockingQueue;
33 26 import java.util.concurrent.ThreadPoolExecutor;
... ... @@ -183,13 +176,14 @@ public class DataCaptureTask {
183 176 final Date createTime = obj.getDate("createTime");
184 177 return createTime.compareTo(startTime) >= 0 && createTime.compareTo(endTime) <= 0;
185 178 })
186   - .map(item -> {
  179 + /*.map(item -> {
187 180 JSONObject obj = (JSONObject) item;
188 181 final Integer durationSeconds = obj.getInteger("duration");
189 182 obj.put("duration", BigDecimal.valueOf(durationSeconds).divide(BigDecimal.valueOf(60), 1, RoundingMode.HALF_UP).doubleValue());
190 183 obj.put("startTime", obj.getLong("createTime") - durationSeconds * 1000);
191 184 return obj;
192   - })
  185 + })*/
  186 + .map(item -> (JSONObject) item)
193 187 .collect(Collectors.toList());
194 188 if (!CollectionUtils.isEmpty(collect)) {// 找到回放, 筛选结果不为空
195 189 return collect;
... ...