Commit 1ab15ffb6fbe1e5187937c486a2ad90eaa949c33
1 parent
b1586371
2022年6月1日16:52:12 更改回放匹配策略
Showing
6 changed files
with
49 additions
and
11 deletions
src/main/java/cn/fw/freya/dao/AccountDao.java
@@ -15,7 +15,6 @@ import java.util.List; | @@ -15,7 +15,6 @@ import java.util.List; | ||
15 | */ | 15 | */ |
16 | @Repository | 16 | @Repository |
17 | public interface AccountDao extends JpaRepository<Account, Long> { | 17 | public interface AccountDao extends JpaRepository<Account, Long> { |
18 | - | ||
19 | /** | 18 | /** |
20 | * 删除指定账号 | 19 | * 删除指定账号 |
21 | * | 20 | * |
@@ -98,4 +97,12 @@ public interface AccountDao extends JpaRepository<Account, Long> { | @@ -98,4 +97,12 @@ public interface AccountDao extends JpaRepository<Account, Long> { | ||
98 | @Modifying | 97 | @Modifying |
99 | @Query("update Account a set a.cookiesStatus = ?3 where a.phoneNo = ?1 and a.type = ?2") | 98 | @Query("update Account a set a.cookiesStatus = ?3 where a.phoneNo = ?1 and a.type = ?2") |
100 | void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus); | 99 | void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus); |
100 | + | ||
101 | + /** | ||
102 | + * 设置账户状态为未完成 | ||
103 | + * @param accountNo | ||
104 | + */ | ||
105 | + @Modifying | ||
106 | + @Query("update Account a set a.done = false where a.phoneNo = ?1") | ||
107 | + void setAccountUndone(String accountNo); | ||
101 | } | 108 | } |
src/main/java/cn/fw/freya/service/CrawlBizService.java
@@ -249,6 +249,7 @@ public class CrawlBizService { | @@ -249,6 +249,7 @@ public class CrawlBizService { | ||
249 | final List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息 | 249 | final List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息 |
250 | final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); | 250 | final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); |
251 | if (!reportLive) { | 251 | if (!reportLive) { |
252 | + accountService.setAccountUndone(accountNo); | ||
252 | log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); | 253 | log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); |
253 | } | 254 | } |
254 | } | 255 | } |
src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java
@@ -256,6 +256,7 @@ public class DongCheDiCrawl implements CrawlStrategy { | @@ -256,6 +256,7 @@ public class DongCheDiCrawl implements CrawlStrategy { | ||
256 | */ | 256 | */ |
257 | @Override | 257 | @Override |
258 | public List<LivePool> getYesterdayLiveMsg(String accountNo) { | 258 | public List<LivePool> getYesterdayLiveMsg(String accountNo) { |
259 | + List<LivePool> livePoolList = new ArrayList<>(); | ||
259 | final List<LivePool> hasFoundLive = common.getHasFoundLive(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); | 260 | final List<LivePool> hasFoundLive = common.getHasFoundLive(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); |
260 | if (Objects.nonNull(hasFoundLive)) { | 261 | if (Objects.nonNull(hasFoundLive)) { |
261 | return hasFoundLive; | 262 | return hasFoundLive; |
@@ -277,7 +278,19 @@ public class DongCheDiCrawl implements CrawlStrategy { | @@ -277,7 +278,19 @@ public class DongCheDiCrawl implements CrawlStrategy { | ||
277 | ); | 278 | ); |
278 | String res = RequestUtil.get(config); | 279 | String res = RequestUtil.get(config); |
279 | log.info(String.format("%s [%s]平台账户号为: %s的直播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, res)); | 280 | log.info(String.format("%s [%s]平台账户号为: %s的直播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, res)); |
280 | - final JSONObject response = JSONObject.parseObject(res); | 281 | + JSONObject response = new JSONObject(); |
282 | + try { | ||
283 | + response = JSONObject.parseObject(res); | ||
284 | + } catch (Exception e) { | ||
285 | + log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据"); | ||
286 | + final LivePool nullLive = LivePool.builder() | ||
287 | + .type(this.getType().getValue()) | ||
288 | + .phoneNo(accountNo) | ||
289 | + .reportDate(new Date()) | ||
290 | + .build(); | ||
291 | + livePoolDao.save(nullLive); | ||
292 | + return livePoolList; | ||
293 | + } | ||
281 | if (this.verifyCookies(response)) { | 294 | if (this.verifyCookies(response)) { |
282 | return null; | 295 | return null; |
283 | } | 296 | } |
@@ -287,7 +300,6 @@ public class DongCheDiCrawl implements CrawlStrategy { | @@ -287,7 +300,6 @@ public class DongCheDiCrawl implements CrawlStrategy { | ||
287 | } | 300 | } |
288 | JSONArray roomList = response.getJSONObject("data").getJSONObject("data").getJSONArray("Rooms"); | 301 | JSONArray roomList = response.getJSONObject("data").getJSONObject("data").getJSONArray("Rooms"); |
289 | JSONArray mctLiveDetails = this.getMCTLiveDetails(accountNo, LocalDateTime.of(LocalDate.now().minusDays(1), LocalTime.MIN).toEpochSecond(ZoneOffset.of("+8"))); | 302 | JSONArray mctLiveDetails = this.getMCTLiveDetails(accountNo, LocalDateTime.of(LocalDate.now().minusDays(1), LocalTime.MIN).toEpochSecond(ZoneOffset.of("+8"))); |
290 | - List<LivePool> livePoolList = new ArrayList<>(); | ||
291 | final LocalDate yesterday = LocalDate.now().minusDays(1); | 303 | final LocalDate yesterday = LocalDate.now().minusDays(1); |
292 | final long yesterdaySecond = LocalDateTime.of(yesterday, LocalTime.MIN).toInstant(ZoneOffset.of("+8")).toEpochMilli(); | 304 | final long yesterdaySecond = LocalDateTime.of(yesterday, LocalTime.MIN).toInstant(ZoneOffset.of("+8")).toEpochMilli(); |
293 | roomList.forEach(item -> { | 305 | roomList.forEach(item -> { |
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
@@ -466,9 +466,9 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -466,9 +466,9 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
466 | final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); | 466 | final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); |
467 | // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 | 467 | // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 |
468 | // (liveStartSub < 2)->说明开播时间几乎一样 | 468 | // (liveStartSub < 2)->说明开播时间几乎一样 |
469 | - if (!((Objects.equals(liveDuration, duration) || | ||
470 | - (timeSubAbs < 5)) && liveStartSub < 10) || | ||
471 | - !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 1000 && duration >= durationThreshold)) { | 469 | + if ((!((Objects.equals(liveDuration, duration) || |
470 | + (timeSubAbs < 5)) && liveStartSub < 10)) && | ||
471 | + !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold)) { | ||
472 | playbackMsg = null; | 472 | playbackMsg = null; |
473 | log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); | 473 | log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); |
474 | } | 474 | } |
src/main/java/cn/fw/freya/service/data/AccountService.java
@@ -55,9 +55,17 @@ public interface AccountService { | @@ -55,9 +55,17 @@ public interface AccountService { | ||
55 | /** | 55 | /** |
56 | * 更新账户cookies状态 | 56 | * 更新账户cookies状态 |
57 | * | 57 | * |
58 | - * @param phoneNo | 58 | + * @param accountNo |
59 | * @param type | 59 | * @param type |
60 | * @param cookiesStatus | 60 | * @param cookiesStatus |
61 | */ | 61 | */ |
62 | - void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus); | 62 | + void updateAccountCookiesStatus(String accountNo, Integer type, boolean cookiesStatus); |
63 | + | ||
64 | + /** | ||
65 | + * 设置账户为未完成状态 | ||
66 | + * | ||
67 | + * @param accountNo | ||
68 | + */ | ||
69 | + void setAccountUndone(String accountNo); | ||
70 | + | ||
63 | } | 71 | } |
src/main/java/cn/fw/freya/service/data/impl/AccountServiceImpl.java
@@ -63,12 +63,22 @@ public class AccountServiceImpl implements AccountService { | @@ -63,12 +63,22 @@ public class AccountServiceImpl implements AccountService { | ||
63 | /** | 63 | /** |
64 | * 更新账户cookies状态 | 64 | * 更新账户cookies状态 |
65 | * | 65 | * |
66 | - * @param phoneNo | 66 | + * @param accountNo |
67 | * @param type | 67 | * @param type |
68 | * @param cookiesStatus | 68 | * @param cookiesStatus |
69 | */ | 69 | */ |
70 | @Override | 70 | @Override |
71 | - public void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus) { | ||
72 | - accountDao.updateAccountCookiesStatus(phoneNo, type, cookiesStatus); | 71 | + public void updateAccountCookiesStatus(String accountNo, Integer type, boolean cookiesStatus) { |
72 | + accountDao.updateAccountCookiesStatus(accountNo, type, cookiesStatus); | ||
73 | + } | ||
74 | + | ||
75 | + /** | ||
76 | + * 设置账户为未完成状态 | ||
77 | + * | ||
78 | + * @param accountNo | ||
79 | + */ | ||
80 | + @Override | ||
81 | + public void setAccountUndone(String accountNo) { | ||
82 | + accountDao.setAccountUndone(accountNo); | ||
73 | } | 83 | } |
74 | } | 84 | } |