From 7ae97ee4dd6e3727b55d55b843ffd818a051155d Mon Sep 17 00:00:00 2001 From: 王明元 <97082371@qq.com> Date: Fri, 27 May 2022 14:38:39 +0800 Subject: [PATCH] 2022年5月27日14:37:21 标记有直播回放数据但是回放未匹配上昨天直播的项 --- src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java | 64 +++++++++++++++++++++++++++++++++------------------------------- src/main/java/cn/fw/freya/task/DataCaptureTask.java | 11 ++++++++--- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java index 592becf..5e8809e 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java @@ -447,48 +447,50 @@ public class KuaiShouCrawl implements CrawlStrategy { return livePoolList; } - public JSONObject setPlaybackUrl(LivePool dbLive, List collect) { + public JSONObject setPlaybackUrl(LivePool dbLive, List collect, Double durationThreshold) { JSONObject object = new JSONObject(); object.put("coverUrl", null); object.put("playbackUrl", null); + if (CollectionUtils.isEmpty(collect)) + return object; final String accountNo = dbLive.getPhoneNo(); final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长 final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳 - if (liveDuration >= 60) {// 直播时长大于等于60分钟才记录回放信息 - if (!CollectionUtils.isEmpty(collect)) { - JSONObject playbackMsg; - if (Objects.equals(collect.size(), 1)) { - playbackMsg = collect.get(0); - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 - final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳 + if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息 + JSONObject playbackMsg; + if (Objects.equals(collect.size(), 1)) { + playbackMsg = collect.get(0); + Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 + final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳 + final double timeSubAbs = Math.abs(liveDuration - duration); + final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); + // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 + // (liveStartSub < 2)->说明开播时间几乎一样 + if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) { + playbackMsg = null; + log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); + } + } else { + List collect1 = collect.stream().filter(item1 -> { + Double duration = item1.getDouble("duration"); + final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳 final double timeSubAbs = Math.abs(liveDuration - duration); final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); - // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 - // (liveStartSub < 2)->说明开播时间几乎一样 - if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) { - playbackMsg = null; - log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); - } + return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10; + }).collect(Collectors.toList()); + if (Objects.equals(collect1.size(), 1)) { + playbackMsg = collect1.get(0); } else { - List collect1 = collect.stream().filter(item1 -> { - Double duration = item1.getDouble("duration"); - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳 - final double timeSubAbs = Math.abs(liveDuration - duration); - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10; - }).collect(Collectors.toList()); - if (Objects.equals(collect1.size(), 1)) { - playbackMsg = collect1.get(0); - } else { - playbackMsg = null; - log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); - } - } - if (Objects.nonNull(playbackMsg)) { - object.put("coverUrl", playbackMsg.getString("coverUrl")); - object.put("playbackUrl", this.playbackBaseUrl + playbackMsg.getString("productId")); + playbackMsg = null; + log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); } } + if (Objects.nonNull(playbackMsg)) { + object.put("coverUrl", playbackMsg.getString("coverUrl")); + object.put("playbackUrl", this.playbackBaseUrl + playbackMsg.getString("productId")); + } else { + object.put("hasInvalidPlayback", true); + } } return object; } diff --git a/src/main/java/cn/fw/freya/task/DataCaptureTask.java b/src/main/java/cn/fw/freya/task/DataCaptureTask.java index 28de263..5b8e6c7 100644 --- a/src/main/java/cn/fw/freya/task/DataCaptureTask.java +++ b/src/main/java/cn/fw/freya/task/DataCaptureTask.java @@ -99,8 +99,9 @@ public class DataCaptureTask { */ @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) public void captureLivePlayback() { + Double durationThreshold = 60d;// 设置直播时长阈值 final Random random = new Random(); - List withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) + List withoutPlaybackLive = common.getWithoutPlaybackLive(1, durationThreshold) .stream() .filter(item -> 0 <= item.getGetPlaybackFailTimes() && item.getGetPlaybackFailTimes() < 30) .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据 @@ -119,11 +120,15 @@ public class DataCaptureTask { final List playbackMsg = this.getPlaybackMsg(account);// 获取该人的直播回放信息 boolean flag = false; for (LivePool item : list) { - JSONObject obj = kuaiShouCrawl.setPlaybackUrl(item, playbackMsg); + JSONObject obj = kuaiShouCrawl.setPlaybackUrl(item, playbackMsg, durationThreshold); String playbackUrl = obj.getString("playbackUrl"); + Boolean hasInvalidPlayback = obj.getBoolean("hasInvalidPlayback"); final LivePool livePool = PublicUtil.copy(item, LivePool.class);// 对象拷贝 if (Objects.isNull(playbackUrl)) { - livePool.setGetPlaybackFailTimes(livePool.getGetPlaybackFailTimes() + 1); + if (hasInvalidPlayback) + livePool.setGetPlaybackFailTimes(-2);// 表示拿到回放数据, 但是没有和昨天直播匹配的回放 + else + livePool.setGetPlaybackFailTimes(livePool.getGetPlaybackFailTimes() + 1); } else { flag = true; livePool.setRoomCoverImage(obj.getString("coverUrl")); -- libgit2 0.22.2