Commit 1a95281dd5b9d08fa191fd5a00b5fef2808d5542
1 parent
5736d4ca
2023年1月9日15:47:56 修复快手回播无法获取问题
Showing
2 changed files
with
15 additions
and
20 deletions
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
... | ... | @@ -52,7 +52,6 @@ import java.io.FileOutputStream; |
52 | 52 | import java.io.IOException; |
53 | 53 | import java.io.OutputStreamWriter; |
54 | 54 | import java.math.BigDecimal; |
55 | -import java.math.RoundingMode; | |
56 | 55 | import java.net.URLDecoder; |
57 | 56 | import java.nio.charset.StandardCharsets; |
58 | 57 | import java.time.LocalDate; |
... | ... | @@ -612,6 +611,7 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle { |
612 | 611 | } |
613 | 612 | |
614 | 613 | public JSONObject setPlaybackUrl(LivePool dbLive, List<JSONObject> collect, Double durationThreshold) { |
614 | + final int maxSubRangeMinutes = 2;// 直播结束时间与回播创建时间允许最大偏差分钟数 | |
615 | 615 | JSONObject object = new JSONObject(); |
616 | 616 | object.put("coverUrl", null); |
617 | 617 | object.put("playbackUrl", null); |
... | ... | @@ -624,25 +624,26 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle { |
624 | 624 | JSONObject playbackMsg; |
625 | 625 | if (Objects.equals(collect.size(), 1)) { |
626 | 626 | playbackMsg = collect.get(0); |
627 | - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 | |
627 | + /*Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 | |
628 | 628 | final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳 |
629 | 629 | final double timeSubAbs = Math.abs(liveDuration - duration); |
630 | - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); | |
630 | + final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();*/ | |
631 | 631 | // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 |
632 | 632 | // (liveStartSub < 2)->说明开播时间几乎一样 |
633 | - if ((!((Objects.equals(liveDuration, duration) || | |
633 | + /*if ((!((Objects.equals(liveDuration, duration) || | |
634 | 634 | (timeSubAbs < 5)) && liveStartSub < 10)) && |
635 | - !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold)) { | |
635 | + !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold))*/ | |
636 | + if (Math.abs(playbackMsg.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) > maxSubRangeMinutes * 60 * 1000) { | |
636 | 637 | playbackMsg = null; |
637 | 638 | log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); |
638 | 639 | } |
639 | 640 | } else { |
640 | - List<JSONObject> collect1 = collect.stream().filter(item1 -> { | |
641 | - Double duration = item1.getDouble("duration"); | |
642 | - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳 | |
643 | - final double timeSubAbs = Math.abs(liveDuration - duration); | |
644 | - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); | |
645 | - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10; | |
641 | + List<JSONObject> collect1 = collect.stream().filter(item -> { | |
642 | + //Double duration = item.getDouble("duration"); | |
643 | + //final Long startTimeStamp = item.getLong("startTime");// 回放信息返回直播开始时间戳 | |
644 | + //final double timeSubAbs = Math.abs(liveDuration - duration); | |
645 | + //final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); | |
646 | + return Math.abs(item.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) < maxSubRangeMinutes * 60 * 1000; | |
646 | 647 | }).collect(Collectors.toList()); |
647 | 648 | if (Objects.equals(collect1.size(), 1)) { |
648 | 649 | playbackMsg = collect1.get(0); | ... | ... |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
... | ... | @@ -11,13 +11,8 @@ import cn.fw.freya.service.data.AccountService; |
11 | 11 | import cn.fw.freya.utils.DateUtil; |
12 | 12 | import cn.fw.freya.utils.PublicUtil; |
13 | 13 | import cn.fw.freya.utils.ThreadPoolUtil; |
14 | -import com.alibaba.fastjson.JSON; | |
15 | 14 | import com.alibaba.fastjson.JSONArray; |
16 | 15 | import com.alibaba.fastjson.JSONObject; |
17 | -import com.alibaba.fastjson.annotation.JSONField; | |
18 | -import lombok.AllArgsConstructor; | |
19 | -import lombok.Data; | |
20 | -import lombok.NoArgsConstructor; | |
21 | 16 | import lombok.RequiredArgsConstructor; |
22 | 17 | import lombok.extern.slf4j.Slf4j; |
23 | 18 | import org.springframework.context.ApplicationEventPublisher; |
... | ... | @@ -26,8 +21,6 @@ import org.springframework.stereotype.Component; |
26 | 21 | import org.springframework.util.CollectionUtils; |
27 | 22 | |
28 | 23 | import java.io.IOException; |
29 | -import java.math.BigDecimal; | |
30 | -import java.math.RoundingMode; | |
31 | 24 | import java.util.*; |
32 | 25 | import java.util.concurrent.BlockingQueue; |
33 | 26 | import java.util.concurrent.ThreadPoolExecutor; |
... | ... | @@ -183,13 +176,14 @@ public class DataCaptureTask { |
183 | 176 | final Date createTime = obj.getDate("createTime"); |
184 | 177 | return createTime.compareTo(startTime) >= 0 && createTime.compareTo(endTime) <= 0; |
185 | 178 | }) |
186 | - .map(item -> { | |
179 | + /*.map(item -> { | |
187 | 180 | JSONObject obj = (JSONObject) item; |
188 | 181 | final Integer durationSeconds = obj.getInteger("duration"); |
189 | 182 | obj.put("duration", BigDecimal.valueOf(durationSeconds).divide(BigDecimal.valueOf(60), 1, RoundingMode.HALF_UP).doubleValue()); |
190 | 183 | obj.put("startTime", obj.getLong("createTime") - durationSeconds * 1000); |
191 | 184 | return obj; |
192 | - }) | |
185 | + })*/ | |
186 | + .map(item -> (JSONObject) item) | |
193 | 187 | .collect(Collectors.toList()); |
194 | 188 | if (!CollectionUtils.isEmpty(collect)) {// 找到回放, 筛选结果不为空 |
195 | 189 | return collect; | ... | ... |