From 1a95281dd5b9d08fa191fd5a00b5fef2808d5542 Mon Sep 17 00:00:00 2001 From: 王明元 <97082371@qq.com> Date: Mon, 9 Jan 2023 15:48:18 +0800 Subject: [PATCH] 2023年1月9日15:47:56 修复快手回播无法获取问题 --- src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java | 23 ++++++++++++----------- src/main/java/cn/fw/freya/task/DataCaptureTask.java | 12 +++--------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java index 0bd2264..fc7f828 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java @@ -52,7 +52,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.math.BigDecimal; -import java.math.RoundingMode; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -612,6 +611,7 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle { } public JSONObject setPlaybackUrl(LivePool dbLive, List collect, Double durationThreshold) { + final int maxSubRangeMinutes = 2;// 直播结束时间与回播创建时间允许最大偏差分钟数 JSONObject object = new JSONObject(); object.put("coverUrl", null); object.put("playbackUrl", null); @@ -624,25 +624,26 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle { JSONObject playbackMsg; if (Objects.equals(collect.size(), 1)) { playbackMsg = collect.get(0); - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 + /*Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳 final double timeSubAbs = Math.abs(liveDuration - duration); - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); + final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();*/ // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 // (liveStartSub < 2)->说明开播时间几乎一样 - if ((!((Objects.equals(liveDuration, duration) || + /*if ((!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) && - !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold)) { + !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold))*/ + if (Math.abs(playbackMsg.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) > maxSubRangeMinutes * 60 * 1000) { playbackMsg = null; log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); } } else { - List collect1 = collect.stream().filter(item1 -> { - Double duration = item1.getDouble("duration"); - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳 - final double timeSubAbs = Math.abs(liveDuration - duration); - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10; + List collect1 = collect.stream().filter(item -> { + //Double duration = item.getDouble("duration"); + //final Long startTimeStamp = item.getLong("startTime");// 回放信息返回直播开始时间戳 + //final double timeSubAbs = Math.abs(liveDuration - duration); + //final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); + return Math.abs(item.getLong("createTime") - liveStartTimeStamp - liveDuration * 60 * 1000) < maxSubRangeMinutes * 60 * 1000; }).collect(Collectors.toList()); if (Objects.equals(collect1.size(), 1)) { playbackMsg = collect1.get(0); diff --git a/src/main/java/cn/fw/freya/task/DataCaptureTask.java b/src/main/java/cn/fw/freya/task/DataCaptureTask.java index cf938f0..3fc9e1a 100644 --- a/src/main/java/cn/fw/freya/task/DataCaptureTask.java +++ b/src/main/java/cn/fw/freya/task/DataCaptureTask.java @@ -11,13 +11,8 @@ import cn.fw.freya.service.data.AccountService; import cn.fw.freya.utils.DateUtil; import cn.fw.freya.utils.PublicUtil; import cn.fw.freya.utils.ThreadPoolUtil; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.annotation.JSONField; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; @@ -26,8 +21,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -183,13 +176,14 @@ public class DataCaptureTask { final Date createTime = obj.getDate("createTime"); return createTime.compareTo(startTime) >= 0 && createTime.compareTo(endTime) <= 0; }) - .map(item -> { + /*.map(item -> { JSONObject obj = (JSONObject) item; final Integer durationSeconds = obj.getInteger("duration"); obj.put("duration", BigDecimal.valueOf(durationSeconds).divide(BigDecimal.valueOf(60), 1, RoundingMode.HALF_UP).doubleValue()); obj.put("startTime", obj.getLong("createTime") - durationSeconds * 1000); return obj; - }) + })*/ + .map(item -> (JSONObject) item) .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(collect)) {// 找到回放, 筛选结果不为空 return collect; -- libgit2 0.22.2