From d472478d333c0cbbca7312129a8834d36de1962c Mon Sep 17 00:00:00 2001 From: 王明元 <97082371@qq.com> Date: Thu, 30 Jun 2022 18:20:48 +0800 Subject: [PATCH] 2022年6月30日18:19:28 调用接口时候发现500002错误码, 直接更新sig信息 --- src/main/java/cn/fw/freya/controller/BLBLController.java | 5 +++-- src/main/java/cn/fw/freya/controller/DCDController.java | 5 +++-- src/main/java/cn/fw/freya/controller/DYController.java | 5 +++-- src/main/java/cn/fw/freya/controller/KSController.java | 23 ++++++++++++++--------- src/main/java/cn/fw/freya/controller/OtherController.java | 17 +++++++++-------- src/main/java/cn/fw/freya/dao/AccountDao.java | 38 ++++++++++++++++++++------------------ src/main/java/cn/fw/freya/dao/CookieDao.java | 16 ++++++++-------- src/main/java/cn/fw/freya/dao/LivePoolDao.java | 8 ++++---- src/main/java/cn/fw/freya/dao/VideoPoolDao.java | 12 ++++++------ src/main/java/cn/fw/freya/enums/AccountTypeEnum.java | 2 +- src/main/java/cn/fw/freya/model/data/Account.java | 10 +++++----- src/main/java/cn/fw/freya/model/data/FwCookie.java | 12 ++++++------ src/main/java/cn/fw/freya/model/data/LiveOverview.java | 4 ++-- src/main/java/cn/fw/freya/model/data/pool/LivePool.java | 4 ++-- src/main/java/cn/fw/freya/model/data/pool/VideoPool.java | 4 ++-- src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java | 2 +- src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java | 43 +++++++++++++++++++++++++++++++++++++++++++ src/main/java/cn/fw/freya/service/CommonBizService.java | 2 +- src/main/java/cn/fw/freya/service/CrawlBizService.java | 62 ++++++++++++++++++++++++++++++++------------------------------ src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java | 5 +++-- src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java | 58 ++++++++++++++++++++++++++++++++++------------------------ src/main/java/cn/fw/freya/service/crawl/impl/Common.java | 29 +++++++++++++++++------------ src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java | 60 +++++++++++++++++++++++++++++++++++++----------------------- src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java | 87 +++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------- src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------------------------------- src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java | 12 ++++++------ src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java | 14 ++++++++------ src/main/java/cn/fw/freya/task/DataCaptureTask.java | 17 ++++++++--------- src/main/java/cn/fw/freya/utils/RequestUtil.java | 3 --- src/main/resources/NS_sig3Msg.properties | 3 +++ src/main/resources/alert.sql | 19 +++++++++++++++++++ 31 files changed, 520 insertions(+), 315 deletions(-) create mode 100644 src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java create mode 100644 src/main/resources/NS_sig3Msg.properties create mode 100644 src/main/resources/alert.sql diff --git a/src/main/java/cn/fw/freya/controller/BLBLController.java b/src/main/java/cn/fw/freya/controller/BLBLController.java index 4d83903..0c6f0e7 100644 --- a/src/main/java/cn/fw/freya/controller/BLBLController.java +++ b/src/main/java/cn/fw/freya/controller/BLBLController.java @@ -2,6 +2,7 @@ package cn.fw.freya.controller; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.impl.BilibiliCrawl; import lombok.RequiredArgsConstructor; import org.springframework.validation.annotation.Validated; @@ -52,8 +53,8 @@ public class BLBLController { * @param accountNo 账户号 */ @GetMapping("/updateAccountFans") - public Integer updateAccountFans(String accountNo) throws IOException { - return bilibiliCrawl.updateAccountFans(accountNo); + public ReportAccountDto updateAccountFans(String accountNo) throws IOException { + return bilibiliCrawl.updateAccountMsg(accountNo); } /** diff --git a/src/main/java/cn/fw/freya/controller/DCDController.java b/src/main/java/cn/fw/freya/controller/DCDController.java index 6102edf..859d537 100644 --- a/src/main/java/cn/fw/freya/controller/DCDController.java +++ b/src/main/java/cn/fw/freya/controller/DCDController.java @@ -2,6 +2,7 @@ package cn.fw.freya.controller; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.impl.DongCheDiCrawl; import lombok.RequiredArgsConstructor; import org.springframework.validation.annotation.Validated; @@ -61,8 +62,8 @@ public class DCDController { * @param accountNo 账户号 */ @GetMapping("/updateAccountFans") - public Integer updateAccountFans(String accountNo) { - return dongCheDiCrawl.updateAccountFans(accountNo); + public ReportAccountDto updateAccountFans(String accountNo) { + return dongCheDiCrawl.updateAccountMsg(accountNo); } /** diff --git a/src/main/java/cn/fw/freya/controller/DYController.java b/src/main/java/cn/fw/freya/controller/DYController.java index db3dc4a..e4ea7a0 100644 --- a/src/main/java/cn/fw/freya/controller/DYController.java +++ b/src/main/java/cn/fw/freya/controller/DYController.java @@ -3,6 +3,7 @@ package cn.fw.freya.controller; import cn.fw.freya.model.data.LiveOverview; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.impl.DouYinCrawl; import lombok.RequiredArgsConstructor; import org.springframework.validation.annotation.Validated; @@ -62,8 +63,8 @@ public class DYController { * @param accountNo 账户号 */ @GetMapping("/updateAccountFans") - public Integer updateAccountFans(String accountNo) { - return douyinCrawl.updateAccountFans(accountNo); + public ReportAccountDto updateAccountFans(String accountNo) { + return douyinCrawl.updateAccountMsg(accountNo); } /** diff --git a/src/main/java/cn/fw/freya/controller/KSController.java b/src/main/java/cn/fw/freya/controller/KSController.java index 96d545b..af94636 100644 --- a/src/main/java/cn/fw/freya/controller/KSController.java +++ b/src/main/java/cn/fw/freya/controller/KSController.java @@ -2,6 +2,7 @@ package cn.fw.freya.controller; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.impl.KuaiShouCrawl; import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; @@ -59,8 +60,8 @@ public class KSController { * @param accountNo 账户号 */ @GetMapping("/updateAccountFans") - public Integer updateAccountFans(String accountNo) throws IOException { - return kuaishouCrawl.updateAccountFans(accountNo); + public ReportAccountDto updateAccountFans(String accountNo) throws IOException { + return kuaishouCrawl.updateAccountMsg(accountNo); } /** @@ -79,13 +80,12 @@ public class KSController { * @param accountNo 账户号 */ @GetMapping("/getNS_sig3") - public String getNS_sig3(String accountNo, Integer type) { - return kuaishouCrawl.getNS_sig3(accountNo, type); + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) { + return kuaishouCrawl.getNS_sig3(accountNo, type, retryGet); } /** * 设置setSig3Map - * */ @GetMapping("/setSig3Map") public boolean setSig3Map() { @@ -94,16 +94,14 @@ public class KSController { /** * 设置setSig3Map - * */ @PostMapping("/setMapFromString") public boolean setMapFromString(@RequestBody String jsonStr) { - return kuaishouCrawl.setMapFromString(jsonStr); + return kuaishouCrawl.setMapFromString(jsonStr, false); } /** * 停止设置setSig3Map - * */ @GetMapping("/stopSetSig3Map") public boolean stopSetSig3Map() { @@ -111,8 +109,15 @@ public class KSController { } /** + * 清空Sig3Map + */ + @GetMapping("/cleanSig3Map") + public boolean cleanSig3Map() { + return kuaishouCrawl.cleanSig3Map(); + } + + /** * 获取getSig3Map - * */ @GetMapping("/getSig3Map") public String getSig3Map() { diff --git a/src/main/java/cn/fw/freya/controller/OtherController.java b/src/main/java/cn/fw/freya/controller/OtherController.java index 7887694..f378ae8 100644 --- a/src/main/java/cn/fw/freya/controller/OtherController.java +++ b/src/main/java/cn/fw/freya/controller/OtherController.java @@ -5,6 +5,7 @@ import cn.fw.freya.model.data.Account; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.dto.other.ReportLiveDto; import cn.fw.freya.model.dto.other.ReportVideoDto; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.CommonBizService; import cn.fw.freya.service.crawl.impl.Common; import cn.fw.freya.service.rpc.AccountRpcService; @@ -86,16 +87,16 @@ public class OtherController { } /** - * 上报账户粉丝数 + * 上报账户信息 * - * @param account 账号 - * @param type 账号类型 - * @param fansCnt 粉丝数 + * @param account 账号 + * @param type 账号类型 + * @param accountMsg 账户信息 * @return */ - @PostMapping("/reportFansCnt") - public ResponseMessage reportFansCnt(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "粉丝数量不能为空") Integer fansCnt) { - return ResponseMessage.success(reportRpcService.reportFansCnt(account, type, fansCnt)); + @PostMapping("/reportAccountMsg") + public ResponseMessage reportAccountMsg(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "账户信息不能为空") ReportAccountDto accountMsg) { + return ResponseMessage.success(reportRpcService.reportAccountMsg(account, type, accountMsg)); } /** @@ -175,7 +176,7 @@ public class OtherController { * @return */ @GetMapping("/getWithoutPlaybackLive") - public List getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空")Double durationThreshold) { + public List getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空") Double durationThreshold) { return common.getWithoutPlaybackLive(type, durationThreshold); } diff --git a/src/main/java/cn/fw/freya/dao/AccountDao.java b/src/main/java/cn/fw/freya/dao/AccountDao.java index 429b8e6..5be10ba 100644 --- a/src/main/java/cn/fw/freya/dao/AccountDao.java +++ b/src/main/java/cn/fw/freya/dao/AccountDao.java @@ -18,23 +18,23 @@ public interface AccountDao extends JpaRepository { /** * 删除指定账号 * - * @param phoneNo + * @param accountNo * @param value */ @Transactional(rollbackFor = Exception.class) @Modifying - @Query("delete from Account a where a.phoneNo = ?1 and a.type = ?2") - void deleteByPhoneNoAndType(String phoneNo, Integer value); + @Query("delete from Account a where a.accountNo = ?1 and a.type = ?2") + void deleteByAccountNoAndType(String accountNo, Integer value); /** * 查询账号 * - * @param phoneNo + * @param accountNo * @param type * @return */ - @Query("select a from Account a where a.phoneNo = ?1 and a.type = ?2 ") - Account findByPhoneNoAndType(String phoneNo, Integer type); + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 ") + Account findByAccountNoAndType(String accountNo, Integer type); /** * 随机获取一个账号 @@ -72,45 +72,47 @@ public interface AccountDao extends JpaRepository { List findNotUseAccount(); /** - * 更新账户粉丝数 + * 更新账户信息 * - * @param id - * @param fansCnt + * @param id 账户id + * @param accountName 账户名 + * @param fansCnt 账户粉丝数 * @return */ @Transactional(rollbackFor = Exception.class) @Modifying - @Query("update Account a set a.fansCnt = ?2, a.reportDate = ?3 where a.id = ?1") - int updateFans(Long id, int fansCnt, Date reportDate); + @Query("update Account a set a.accountName = ?2, a.fansCnt = ?3, a.reportDate = ?4 where a.id = ?1") + int updateMsg(Long id, String accountName, int fansCnt, Date reportDate); /** * 获取今天上报的粉丝数据 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @param type 账户类型 * @param reportDate 上报日期 * @return */ - @Query("select a.fansCnt from Account a where a.phoneNo = ?1 and a.type = ?2 and a.reportDate >= ?3") - Integer getHasReportDate(String phoneNo, Integer type, Date reportDate); + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 and a.reportDate >= ?3") + List getHasReportDate(String accountNo, Integer type, Date reportDate); /** * 更新账户cookies状态 * - * @param phoneNo + * @param accountNo * @param type * @param cookiesStatus */ @Transactional(rollbackFor = Exception.class) @Modifying - @Query("update Account a set a.cookiesStatus = ?3 where a.phoneNo = ?1 and a.type = ?2") - void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus); + @Query("update Account a set a.cookiesStatus = ?3 where a.accountNo = ?1 and a.type = ?2") + void updateAccountCookiesStatus(String accountNo, Integer type, boolean cookiesStatus); /** * 设置账户状态为未完成 + * * @param accountNo */ @Modifying - @Query("update Account a set a.done = false where a.phoneNo = ?1") + @Query("update Account a set a.done = false where a.accountNo = ?1") void setAccountUndone(String accountNo); } diff --git a/src/main/java/cn/fw/freya/dao/CookieDao.java b/src/main/java/cn/fw/freya/dao/CookieDao.java index 52cba1e..856df6e 100644 --- a/src/main/java/cn/fw/freya/dao/CookieDao.java +++ b/src/main/java/cn/fw/freya/dao/CookieDao.java @@ -17,21 +17,21 @@ public interface CookieDao extends JpaRepository { /** * 删除账号对应的cookie * - * @param phoneNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) */ @Transactional @Modifying - @Query("delete from FwCookie f where f.phoneNo = ?1 and f.type = ?2") - void deleteByPhoneNoAndType(String phoneNo, Integer type); + @Query("delete from FwCookie f where f.accountNo = ?1 and f.type = ?2") + void deleteByAccountNoAndType(String accountNo, Integer type); /** * 查询账号对应平台的cookie * - * @param phoneNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) * @return */ - @Query("select f from FwCookie f where f.phoneNo = ?1 and f.type = ?2") - List findByPhoneNoAndType(String phoneNo, Integer type); + @Query("select f from FwCookie f where f.accountNo = ?1 and f.type = ?2") + List findByAccountNoAndType(String accountNo, Integer type); } diff --git a/src/main/java/cn/fw/freya/dao/LivePoolDao.java b/src/main/java/cn/fw/freya/dao/LivePoolDao.java index 0e3db5f..be67443 100644 --- a/src/main/java/cn/fw/freya/dao/LivePoolDao.java +++ b/src/main/java/cn/fw/freya/dao/LivePoolDao.java @@ -17,13 +17,13 @@ public interface LivePoolDao extends JpaRepository { /** * 获取今天上报的数据 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @param type 账户类型 * @param reportDate 上报日期 * @return */ - @Query("select live from LivePool live where live.phoneNo = ?1 and live.type = ?2 and live.reportDate >= ?3") - List getHasReportDate(String phoneNo, Integer type, Date reportDate); + @Query("select live from LivePool live where live.accountNo = ?1 and live.type = ?2 and live.reportDate >= ?3") + List getHasReportData(String accountNo, Integer type, Date reportDate); /** * 获取没有拿到回播的直播 @@ -41,6 +41,6 @@ public interface LivePoolDao extends JpaRepository { * @param accountNo 账户号 * @return */ - @Query("select live from LivePool live where live.phoneNo = ?1") + @Query("select live from LivePool live where live.accountNo = ?1") List getLiveByAccountNo(String accountNo); } diff --git a/src/main/java/cn/fw/freya/dao/VideoPoolDao.java b/src/main/java/cn/fw/freya/dao/VideoPoolDao.java index 61ee7e2..bf83e75 100644 --- a/src/main/java/cn/fw/freya/dao/VideoPoolDao.java +++ b/src/main/java/cn/fw/freya/dao/VideoPoolDao.java @@ -21,24 +21,24 @@ public interface VideoPoolDao extends JpaRepository { /** * 删除 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @param previousDay 前一天 * @param type 账户类型 * @param resourceType 资源类型 */ @Transactional(rollbackFor = Exception.class) @Modifying - @Query("delete from VideoPool where phoneNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4") - void deleteByPhoneNoAndDate(String phoneNo, Date previousDay, Integer type, Integer resourceType); + @Query("delete from VideoPool where accountNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4") + void deleteByAccountNoAndDate(String accountNo, Date previousDay, Integer type, Integer resourceType); /** * 获取今天上报的数据 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @param type 账户类型 * @param reportDate 上报日期 * @return */ - @Query("select v from VideoPool v where v.phoneNo = ?1 and v.type = ?2 and v.reportDate >= ?3") - List getHasReportDate(String phoneNo, Integer type, Date reportDate); + @Query("select v from VideoPool v where v.accountNo = ?1 and v.type = ?2 and v.reportDate >= ?3") + List getHasReportDate(String accountNo, Integer type, Date reportDate); } diff --git a/src/main/java/cn/fw/freya/enums/AccountTypeEnum.java b/src/main/java/cn/fw/freya/enums/AccountTypeEnum.java index 4e699a2..92a0ddd 100644 --- a/src/main/java/cn/fw/freya/enums/AccountTypeEnum.java +++ b/src/main/java/cn/fw/freya/enums/AccountTypeEnum.java @@ -11,7 +11,7 @@ public enum AccountTypeEnum { KS(1, "快手"),// 快手 DY(2, "抖音"),// 抖音 DCD(3, "懂车帝"),// 懂车帝 - BILIBILI(4, "哔哩哔哩动画"),// 哔哩哔哩动画 + BILIBILI(4, "哔哩哔哩"),// 哔哩哔哩 ; @Getter private final Integer value; diff --git a/src/main/java/cn/fw/freya/model/data/Account.java b/src/main/java/cn/fw/freya/model/data/Account.java index 1faa519..85c50e3 100644 --- a/src/main/java/cn/fw/freya/model/data/Account.java +++ b/src/main/java/cn/fw/freya/model/data/Account.java @@ -18,16 +18,12 @@ import java.util.Objects; @AllArgsConstructor @Builder(toBuilder = true) @Entity -@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "type"})) +@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type"})) public class Account { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; /** - * 账户号 - */ - private String phoneNo; - /** * 回播搜索关键词 */ private String playbackSearchKey; @@ -36,6 +32,10 @@ public class Account { */ private String accountNo; /** + * 账户名 + */ + private String accountName; + /** * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) */ private Integer type; diff --git a/src/main/java/cn/fw/freya/model/data/FwCookie.java b/src/main/java/cn/fw/freya/model/data/FwCookie.java index 82fdb4d..71ae388 100644 --- a/src/main/java/cn/fw/freya/model/data/FwCookie.java +++ b/src/main/java/cn/fw/freya/model/data/FwCookie.java @@ -23,7 +23,7 @@ import java.util.Objects; @ToString @AllArgsConstructor @NoArgsConstructor -@Table(name = "cookie", indexes = {@Index(columnList = "phoneNo")}) +@Table(name = "cookie", indexes = {@Index(columnList = "accountNo")}) public class FwCookie { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -31,7 +31,7 @@ public class FwCookie { /** * 账户号 */ - private String phoneNo; + private String accountNo; /** * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) */ @@ -104,13 +104,13 @@ public class FwCookie { /** * WebDriver获取到的cookies->DB * - * @param cookie WebDriver获取到的cookie - * @param phoneNo 账户号 + * @param cookie WebDriver获取到的cookie + * @param accountNo 账户号 * @return cookies */ - public static FwCookie toDb(Cookie cookie, String phoneNo, Integer type) { + public static FwCookie toDb(Cookie cookie, String accountNo, Integer type) { FwCookie fwCookie = new FwCookie(); - fwCookie.setPhoneNo(phoneNo); + fwCookie.setAccountNo(accountNo); fwCookie.setType(type); fwCookie.setDomain(cookie.getDomain()); fwCookie.setExpiryDate(DateUtil.date2LocalDateTime(cookie.getExpiry())); diff --git a/src/main/java/cn/fw/freya/model/data/LiveOverview.java b/src/main/java/cn/fw/freya/model/data/LiveOverview.java index d97bd8d..45b95c6 100644 --- a/src/main/java/cn/fw/freya/model/data/LiveOverview.java +++ b/src/main/java/cn/fw/freya/model/data/LiveOverview.java @@ -18,7 +18,7 @@ import java.util.Date; @Builder(toBuilder = true) @Entity @ToString -@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "reportDate", "type"})) +@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "reportDate", "type"})) public class LiveOverview { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -41,7 +41,7 @@ public class LiveOverview { /** * 账户号 */ - private String phoneNo; + private String accountNo; /** * 直播场次 */ diff --git a/src/main/java/cn/fw/freya/model/data/pool/LivePool.java b/src/main/java/cn/fw/freya/model/data/pool/LivePool.java index 6194886..3934401 100644 --- a/src/main/java/cn/fw/freya/model/data/pool/LivePool.java +++ b/src/main/java/cn/fw/freya/model/data/pool/LivePool.java @@ -20,7 +20,7 @@ import java.util.Objects; @AllArgsConstructor @Builder(toBuilder = true) @Entity -@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "phoneNo", "roomId"})) +@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "accountNo", "roomId"})) public class LivePool { /** * 主键id @@ -35,7 +35,7 @@ public class LivePool { /** * 账户号 */ - private String phoneNo; + private String accountNo; /** * 数据上报日期 */ diff --git a/src/main/java/cn/fw/freya/model/data/pool/VideoPool.java b/src/main/java/cn/fw/freya/model/data/pool/VideoPool.java index 0f22f49..a26a6c3 100644 --- a/src/main/java/cn/fw/freya/model/data/pool/VideoPool.java +++ b/src/main/java/cn/fw/freya/model/data/pool/VideoPool.java @@ -19,7 +19,7 @@ import java.util.Date; @Builder(toBuilder = true) @Entity @ToString -@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "videoId", "type", "reportDate"})) +@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "videoId", "type", "reportDate"})) public class VideoPool { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -27,7 +27,7 @@ public class VideoPool { /** * 账户号 */ - private String phoneNo; + private String accountNo; /** * 视频id **/ diff --git a/src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java b/src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java index c991b16..d33b644 100644 --- a/src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java +++ b/src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java @@ -12,7 +12,7 @@ public class AccountDto { /** * 账号 */ - private String phoneNo; + private String accountNo; /** * 回播搜索关键词 */ diff --git a/src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java b/src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java new file mode 100644 index 0000000..552995f --- /dev/null +++ b/src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java @@ -0,0 +1,43 @@ +package cn.fw.freya.model.dto.rpc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author unknown + * @version 1.0 + * @date 2022/6/29 16:10 + * @Description + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder(toBuilder = true) +public class ReportAccountDto { + /** + * 粉丝数 + */ + private Integer fansCnt; + /** + * 关注数 + */ + private Integer followCnt; + /** + * 点赞数 + */ + private Integer likeCnt; + /** + * 用户在平台的其他信息id + */ + private String userOtherId; + /** + * 用户id + */ + private String accountId; + /** + * 用户名 + */ + private String accountName; +} diff --git a/src/main/java/cn/fw/freya/service/CommonBizService.java b/src/main/java/cn/fw/freya/service/CommonBizService.java index 2050aec..ca918d5 100644 --- a/src/main/java/cn/fw/freya/service/CommonBizService.java +++ b/src/main/java/cn/fw/freya/service/CommonBizService.java @@ -113,7 +113,7 @@ public class CommonBizService { } accountList.addAll(list.stream() .map(item -> Account.builder() - .phoneNo(item.getPhoneNo()) + .accountNo(item.getAccountNo()) .playbackSearchKey(item.getPlaybackSearchKey()) .type(typeEnum.getValue()) .cookiesStatus(Boolean.TRUE) diff --git a/src/main/java/cn/fw/freya/service/CrawlBizService.java b/src/main/java/cn/fw/freya/service/CrawlBizService.java index b109fb0..88dccec 100644 --- a/src/main/java/cn/fw/freya/service/CrawlBizService.java +++ b/src/main/java/cn/fw/freya/service/CrawlBizService.java @@ -4,6 +4,7 @@ import cn.fw.freya.enums.AccountTypeEnum; import cn.fw.freya.model.data.Account; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.CrawlStrategy; import cn.fw.freya.service.data.AccountService; import cn.fw.freya.service.rpc.AccountRpcService; @@ -53,43 +54,43 @@ public class CrawlBizService { /** * 登陆准备 * - * @param phoneNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) * @return */ - public String preLogin(String phoneNo, Integer type) { + public String preLogin(String accountNo, Integer type) { AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); - return crawlStrategy.preLogin(phoneNo); + return crawlStrategy.preLogin(accountNo); } /** * 登陆 * - * @param phoneNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) * @return */ - public boolean doLogin(String phoneNo, Integer type) { + public boolean doLogin(String accountNo, Integer type) { AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); - return crawlStrategy.doLogin(phoneNo); + return crawlStrategy.doLogin(accountNo); } /** * 退出浏览器 * - * @param phoneNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) * @return */ - public boolean exitBrowser(String phoneNo, Integer type) { + public boolean exitBrowser(String accountNo, Integer type) { AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); - return crawlStrategy.exitBrowser(phoneNo); + return crawlStrategy.exitBrowser(accountNo); } /** @@ -99,13 +100,13 @@ public class CrawlBizService { */ public void crawlData(Account account) throws IOException { Integer type = account.getType(); - final String accountNo = account.getPhoneNo(); + final String accountNo = account.getAccountNo(); AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); // 抓取数据 log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据"); - final Integer fansCnt = crawlStrategy.updateAccountFans(accountNo);// 更新粉丝数 - if (Objects.isNull(fansCnt)) { + final ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数 + if (Objects.isNull(accountMsg)) { doPushExpireAccount(account); return; } @@ -120,19 +121,20 @@ public class CrawlBizService { return; } // 上报数据 - final boolean reportFansCnt = this.doReportFansCnt(account, fansCnt); - if (!reportFansCnt) { - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的粉丝数据失败"); + final boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg); + final String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo); + if (!reportAccountMsg) { + log.error(format + "的账户信息失败"); return; } final boolean reportVideo = this.doReportVideo(account, allVideoMsg); if (!reportVideo) { - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的视频数据失败"); + log.error(format + "的视频数据失败"); return; } final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); if (!reportLive) { - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); + log.error(format + "的直播数据失败"); return; } this.afterCrawl(accountService.findById(account.getId())); @@ -158,24 +160,24 @@ public class CrawlBizService { int hasTryTimes = 0; int maxTryTimes = 2; while (!result && hasTryTimes < maxTryTimes) { - result = accountRpcService.pushExpireAccount(account.getPhoneNo(), account.getType()); + result = accountRpcService.pushExpireAccount(account.getAccountNo(), account.getType()); hasTryTimes++; } } /** - * 上报粉丝数 + * 上报账户信息 * - * @param account 账户 - * @param fansCnt 粉丝数 + * @param account 账户 + * @param accountMsg 账户其他信息 * @return */ - private boolean doReportFansCnt(Account account, Integer fansCnt) { + private boolean doReportAccountMsg(Account account, ReportAccountDto accountMsg) { boolean reportFansCnt; int hasTryTimes = 0; int maxTryTimes = 2; while (hasTryTimes < maxTryTimes) { - reportFansCnt = reportRpcService.reportFansCnt(account.getPhoneNo(), account.getType(), fansCnt); + reportFansCnt = reportRpcService.reportAccountMsg(account.getAccountNo(), account.getType(), accountMsg); if (reportFansCnt) { return true; } else { @@ -198,7 +200,7 @@ public class CrawlBizService { int hasTryTimes = 0; int maxTryTimes = 2; while (hasTryTimes < maxTryTimes) { - reportVideo = reportRpcService.reportVideo(account.getPhoneNo(), account.getType(), allVideoMsg); + reportVideo = reportRpcService.reportVideo(account.getAccountNo(), account.getType(), allVideoMsg); if (reportVideo) { return true; } else { @@ -221,7 +223,7 @@ public class CrawlBizService { int hasTryTimes = 0; int maxTryTimes = 2; while (hasTryTimes < maxTryTimes) { - reportLive = reportRpcService.reportLive(account.getPhoneNo(), account.getType(), yesterdayLiveMsg); + reportLive = reportRpcService.reportLive(account.getAccountNo(), account.getType(), yesterdayLiveMsg); if (reportLive) { return true; } else { @@ -243,14 +245,14 @@ public class CrawlBizService { public void reportLive(Account account) throws IOException { log.info(Thread.currentThread().getName() + " spring监听器在CrawlBizService.reportLive()方法上成功收到消息: " + account); Integer type = account.getType(); - final String accountNo = account.getPhoneNo(); + final String accountNo = account.getAccountNo(); AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); final List yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); if (!reportLive) { accountService.setAccountUndone(accountNo); - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); + log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + AccountTypeEnum.getNameByValue(account.getType()) + "的直播数据失败"); } } } diff --git a/src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java b/src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java index 4b1fd27..891719d 100644 --- a/src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java +++ b/src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java @@ -3,6 +3,7 @@ package cn.fw.freya.service.crawl; import cn.fw.freya.enums.AccountTypeEnum; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.utils.http.InitUserAgent; import java.io.IOException; @@ -102,11 +103,11 @@ public interface CrawlStrategy { List getYesterdayLiveMsg(String accountNo) throws IOException; /** - * 更新账户粉丝数 + * 更新账户信息 * * @param accountNo 账户号 * @return * @throws IOException */ - Integer updateAccountFans(String accountNo) throws IOException; + ReportAccountDto updateAccountMsg(String accountNo) throws IOException; } diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java index dd32750..99f75ec 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account; import cn.fw.freya.model.data.FwCookie; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.CrawlStrategy; import cn.fw.freya.service.data.AccountService; import cn.fw.freya.utils.DateUtil; @@ -154,7 +155,7 @@ public class BilibiliCrawl implements CrawlStrategy { } } while (currentPage * pageSize < total);// 条件为false, 退出 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray))); - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue()); + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue()); // 视频数据存库 List videoPoolList = new ArrayList<>(); StringBuilder sb = new StringBuilder(); @@ -189,7 +190,7 @@ public class BilibiliCrawl implements CrawlStrategy { .playCount(playCnt)// 播放次数 .likeCount(Optional.ofNullable(stat.getInteger("like")).orElse(0))// 点赞数 .commentCount(Optional.ofNullable(stat.getInteger("reply")).orElse(0))// 评论数 - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .publishTime(new Date(Archive.getLong("ctime") * 1000))// 发布时间 .videoUrl("https://www.bilibili.com/video/" + videoId)// 视频播放地址 @@ -207,7 +208,7 @@ public class BilibiliCrawl implements CrawlStrategy { videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 } else { final VideoPool nullVideo = VideoPool.builder() - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -367,7 +368,7 @@ public class BilibiliCrawl implements CrawlStrategy { */ livePoolList.add(LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .commentUserCnt(Optional.ofNullable(obj.getInteger("danmu_num")).orElse(0))// 评论数 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) @@ -400,7 +401,7 @@ public class BilibiliCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的Bilibili直播数据"); final LivePool nullLive = LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build(); livePoolDao.save(nullLive); @@ -419,10 +420,10 @@ public class BilibiliCrawl implements CrawlStrategy { */ @Override @Transactional - public Integer updateAccountFans(String accountNo) throws IOException { - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); - if (Objects.nonNull(hasFoundFansCnt)) { - return hasFoundFansCnt; + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException { + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); + if (Objects.nonNull(hasFoundAccountMsg)) { + return hasFoundAccountMsg; } HttpConfig config = HttpConfig.custom() .url("https://member.bilibili.com/x/web/index/stat") @@ -442,28 +443,37 @@ public class BilibiliCrawl implements CrawlStrategy { if (!StringUtils.hasText(res)) { throw new BusinessException("调用Bilibili[粉丝]接口失败"); } - assert response != null; - Integer fansCnt; + ReportAccountDto reportAccountDto; try { - fansCnt = response.getJSONObject("data").getInteger("total_fans"); + JSONObject data = response.getJSONObject("data"); + reportAccountDto = ReportAccountDto.builder() + .fansCnt(data.getInteger("total_fans")) + //.followCnt(data.getInteger("followCnt")) + .likeCnt(data.getInteger("total_like")) + //.accountId(data.getString("userId")) + //.userOtherId(data.getString("userKwaiId")) + //.accountName(data.getString("userName")) + .build(); } catch (Exception e) { log.error("获取[哔哩哔哩]fansCnt发生错误", e); - fansCnt = 0; + reportAccountDto = ReportAccountDto.builder() + .fansCnt(0) + .build(); } - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue()); - if (Objects.nonNull(account)) { - accountDao.updateFans(account.getId(), fansCnt, new Date()); - } else { + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue()); + if (Objects.nonNull(account)) + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date()); + else accountDao.save(Account.builder() .cookiesStatus(true) - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) - .fansCnt(fansCnt) + .fansCnt(reportAccountDto.getFansCnt()) + .accountName(reportAccountDto.getAccountName()) .reportDate(new Date()) .done(false) .build()); - } - return fansCnt; + return reportAccountDto; } /** @@ -563,11 +573,11 @@ public class BilibiliCrawl implements CrawlStrategy { /** * 根据账户号查到用户cookies, 并组装成String形式返回 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @return */ - public String getUserCookies(String phoneNo) { - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue())); + public String getUserCookies(String accountNo) { + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue())); } /** diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/Common.java b/src/main/java/cn/fw/freya/service/crawl/impl/Common.java index 3381ed8..4a9fa68 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/Common.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/Common.java @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.FwCookie; import cn.fw.freya.model.data.ResponseReceived; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; @@ -159,7 +160,7 @@ public class Common { * @param type 账户类型 */ public void deleteCookies(String accountNo, Integer type) { - cookieDao.deleteByPhoneNoAndType(accountNo, type); + cookieDao.deleteByAccountNoAndType(accountNo, type); } /** @@ -169,7 +170,7 @@ public class Common { * @return */ public List loadCookie(String accountNo, Integer type) { - return Optional.ofNullable(cookieDao.findByPhoneNoAndType(accountNo, type)) + return Optional.ofNullable(cookieDao.findByAccountNoAndType(accountNo, type)) .orElse(new ArrayList<>()); } @@ -252,10 +253,14 @@ public class Common { * @param reportDate 指定日期 * @return */ - public Integer getHasFoundFansCnt(String accountNo, Integer type, Date reportDate) { - final Integer hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate); - if (Objects.nonNull(hasReportDate)) { - return hasReportDate; + public ReportAccountDto getHasFoundAccountMsg(String accountNo, Integer type, Date reportDate) { + final List hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate); + if (!CollectionUtils.isEmpty(hasReportDate)) { + final Account account = hasReportDate.get(0); + return ReportAccountDto.builder() + .fansCnt(account.getFansCnt()) + .accountName(account.getAccountName()) + .build(); } return null; } @@ -269,9 +274,9 @@ public class Common { * @return */ public List getHasFoundVideo(String accountNo, Integer type, Date reportDate) { - final List hasReportDate = videoPoolDao.getHasReportDate(accountNo, type, reportDate); - if (hasReportDate.size() > 0) { - return hasReportDate; + final List hasReportData = videoPoolDao.getHasReportDate(accountNo, type, reportDate); + if (hasReportData.size() > 0) { + return hasReportData; } return null; } @@ -285,9 +290,9 @@ public class Common { * @return */ public List getHasFoundLive(String accountNo, Integer type, Date reportDate) { - final List hasReportDate = livePoolDao.getHasReportDate(accountNo, type, reportDate); - if (hasReportDate.size() > 0) { - return hasReportDate; + final List hasReportData = livePoolDao.getHasReportData(accountNo, type, reportDate); + if (hasReportData.size() > 0) { + return hasReportData; } return null; } diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java index 943de5d..ac6b456 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account; import cn.fw.freya.model.data.FwCookie; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.CrawlStrategy; import cn.fw.freya.service.data.AccountService; import cn.fw.freya.utils.DateUtil; @@ -183,7 +184,7 @@ public class DongCheDiCrawl implements CrawlStrategy { assert response1 != null; JSONArray smallVideoJsonArray = Optional.ofNullable(response1.getJSONArray("item_datas")).orElse(new JSONArray());// 获取[小视频]数据数组 videoJsonArray.addAll(smallVideoJsonArray); - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue()); + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue()); // 视频数据存库 List videoPoolList = new ArrayList<>(videoJsonArray.size()); log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray))); @@ -215,7 +216,7 @@ public class DongCheDiCrawl implements CrawlStrategy { .playCount(playCount)// 播放次数 .likeCount(Optional.ofNullable(DCSObj.getInteger("digg_count")).orElse(0))// 点赞数 .commentCount(Optional.ofNullable(DCSObj.getInteger("comment_count")).orElse(0))// 评论数 - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .publishTime(new Date(obj.getLong("create_time") * 1000L))// 发布时间 .videoUrl("https://www.ixigua.com/" + videoId)// 播放地址 @@ -232,7 +233,7 @@ public class DongCheDiCrawl implements CrawlStrategy { videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 } else { final VideoPool nullVideo = VideoPool.builder() - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -285,7 +286,7 @@ public class DongCheDiCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据"); final LivePool nullLive = LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build(); livePoolDao.save(nullLive); @@ -326,7 +327,7 @@ public class DongCheDiCrawl implements CrawlStrategy { assert userInfo != null; livePoolList.add(LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .commentUserCnt(commentCnt)// 评论数 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) @@ -359,7 +360,7 @@ public class DongCheDiCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据"); final LivePool nullLive = LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build(); livePoolDao.save(nullLive); @@ -378,14 +379,19 @@ public class DongCheDiCrawl implements CrawlStrategy { * @return */ @Override - public Integer updateAccountFans(String accountNo) { - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); - if (Objects.nonNull(hasFoundFansCnt)) { - return hasFoundFansCnt; + public ReportAccountDto updateAccountMsg(String accountNo) { + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); + if (Objects.nonNull(hasFoundAccountMsg)) { + return hasFoundAccountMsg; } final String userId; + String userNick; + String ttId; try { - userId = Objects.requireNonNull(this.getUserInfo(accountNo)).getString("userId"); + final JSONObject obj = Objects.requireNonNull(this.getUserInfo(accountNo)); + userId = obj.getString("userId"); + userNick = obj.getString("userNick"); + ttId = obj.getString("ttId"); } catch (NullPointerException e) { return null; } @@ -414,24 +420,32 @@ public class DongCheDiCrawl implements CrawlStrategy { throw new BusinessException("调用懂车帝[粉丝]接口失败"); } final JSONArray data = response1.getJSONArray("data"); + ReportAccountDto reportAccountDto; for (Object item : data) { JSONObject obj = (JSONObject) item; if (obj.getString("title").contains("总数")) { - final Integer fansCnt = obj.getInteger("data"); - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue()); - if (Objects.nonNull(account)) { - accountDao.updateFans(account.getId(), fansCnt, new Date()); - } else { + reportAccountDto = ReportAccountDto.builder() + .fansCnt(obj.getInteger("data")) + //.followCnt(obj.getInteger("followCnt")) + //.likeCnt(obj.getInteger("likeCnt")) + .accountId(userId) + .userOtherId(ttId) + .accountName(userNick) + .build(); + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue()); + if (Objects.nonNull(account)) + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date()); + else accountDao.save(Account.builder() .cookiesStatus(true) - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) - .fansCnt(fansCnt) + .fansCnt(reportAccountDto.getFansCnt()) + .accountName(reportAccountDto.getAccountName()) .reportDate(new Date()) .done(false) .build()); - } - return fansCnt; + return reportAccountDto; } } return null; @@ -559,11 +573,11 @@ public class DongCheDiCrawl implements CrawlStrategy { /** * 根据账户号查到用户cookies, 并组装成String形式返回 * - * @param phoneNo 账户号 + * @param accountNo 账户号 * @return */ - public String getUserCookies(String phoneNo) { - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue())); + public String getUserCookies(String accountNo) { + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue())); } /** diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java index 16ae63c..05b3796 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java @@ -13,6 +13,7 @@ import cn.fw.freya.model.data.LiveOverview; import cn.fw.freya.model.data.ResponseReceived; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.CrawlStrategy; import cn.fw.freya.service.data.AccountService; import cn.fw.freya.utils.DateUtil; @@ -132,13 +133,13 @@ public class DouYinCrawl implements CrawlStrategy { */ @Override @Transactional - /*public List getAllVideoMsg(String phoneNo) { - final List hasFoundVideo = common.getHasFoundVideo(phoneNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); + /*public List getAllVideoMsg(String accountNo) { + final List hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); if (Objects.nonNull(hasFoundVideo)) { return hasFoundVideo; } final String uuid = UUID.randomUUID().toString().replace("-", ""); - final WebDriver driver = this.getDYDriver(phoneNo, uuid); + final WebDriver driver = this.getDYDriver(accountNo, uuid); String targetUrl = "https://creator.douyin.com/creator-micro/content/manage"; driver.get(targetUrl); List endNodes; @@ -146,7 +147,7 @@ public class DouYinCrawl implements CrawlStrategy { do { endNodes = driver.findElements(By.xpath("//div[starts-with(@class,'load-more')]/div[text()='没有更多视频']")); if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) { - this.exitBrowser(phoneNo, uuid); + this.exitBrowser(accountNo, uuid); return null; } if (CollectionUtils.isEmpty(endNodes)) { @@ -211,7 +212,7 @@ public class DouYinCrawl implements CrawlStrategy { .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString()) .duration(jsonObj.getLong("duration") / (1000 * 1.0)) .likeCount(statistics.getInteger("share_count")) - .phoneNo(phoneNo) + .accountNo(accountNo) .playCount(statistics.getInteger("play_count")) .reportDate(new Date()) .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1) @@ -234,7 +235,7 @@ public class DouYinCrawl implements CrawlStrategy { videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 } else { videoPoolDao.save(VideoPool.builder() - .phoneNo(phoneNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -243,9 +244,9 @@ public class DouYinCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息"); } catch (Exception e) { assert log != null; - log.error(LocalDate.now() + " 执行保存" + phoneNo + "的抖音视频数据失败", e); + log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败", e); } - this.exitBrowser(phoneNo, uuid); + this.exitBrowser(accountNo, uuid); return videoPoolList; }*/ public List getAllVideoMsg(String accountNo) { @@ -279,7 +280,7 @@ public class DouYinCrawl implements CrawlStrategy { if (element.getText().contains("没有") || element.getText().contains("起开始展示数据")) { this.exitBrowser(accountNo, uuid); videoPoolDao.save(VideoPool.builder() - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -358,7 +359,7 @@ public class DouYinCrawl implements CrawlStrategy { .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString()) .duration(Optional.ofNullable(jsonObj.getDouble("duration")).orElse(0d) / 1000) .likeCount(Optional.ofNullable(statistics.getInteger("digg_count")).orElse(0)) - .phoneNo(accountNo) + .accountNo(accountNo) .playCount(Optional.ofNullable(statistics.getInteger("play_count")).orElse(0)) .reportDate(new Date()) .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1) @@ -390,7 +391,7 @@ public class DouYinCrawl implements CrawlStrategy { videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 } else { videoPoolDao.save(VideoPool.builder() - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -449,7 +450,7 @@ public class DouYinCrawl implements CrawlStrategy { .type(this.getType().getValue()) .reportDate(new Date()) .uploadTime(new Date()) - .phoneNo(accountNo) + .accountNo(accountNo) .liveCnt(summarizeData.getInteger("yes_live_cnt")) .liveDuration(summarizeData.getDouble("yes_live_duration")) .watchTimesCnt(summarizeData.getInteger("yes_watch_cnt")) @@ -514,7 +515,7 @@ public class DouYinCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的抖音直播数据"); final LivePool nullLive = LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build(); livePoolDao.save(nullLive); @@ -567,7 +568,7 @@ public class DouYinCrawl implements CrawlStrategy { */ livePoolList.add(LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .commentUserCnt(Optional.ofNullable(obj.getInteger("comment_ucnt")).orElse(0))// 评论人数 .consumeUserCnt(Optional.ofNullable(obj.getInteger("consume_ucnt")).orElse(0))// 付费人数 @@ -613,7 +614,7 @@ public class DouYinCrawl implements CrawlStrategy { } else { livePoolDao.save(LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build()); } @@ -633,10 +634,10 @@ public class DouYinCrawl implements CrawlStrategy { */ @Override @Transactional - public Integer updateAccountFans(String accountNo) { - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); - if (Objects.nonNull(hasFoundFansCnt)) { - return hasFoundFansCnt; + public ReportAccountDto updateAccountMsg(String accountNo) { + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); + if (Objects.nonNull(hasFoundAccountMsg)) { + return hasFoundAccountMsg; } final String uuid = UUID.randomUUID().toString().replace("-", ""); final WebDriver driver = this.getDYDriver(accountNo, uuid); @@ -678,9 +679,9 @@ public class DouYinCrawl implements CrawlStrategy { // 读http日志, 处理数据 final List responseReceivedEvents = common.processHttpTransferData(driver); String dataUrl = "https://creator.douyin.com/web/api/media/user/info";// 数据接口地址 - final Integer[] fansCnt = new Integer[1]; + AtomicReference reportAccountDto = new AtomicReference<>(); responseReceivedEvents.forEach(item -> { - JSONObject userInfo; + JSONObject data; try { final HttpResponse response = common.getHttpResponse(driver, item, dataUrl); if (Objects.nonNull(response)) { @@ -690,26 +691,32 @@ public class DouYinCrawl implements CrawlStrategy { if (Objects.isNull(body)) { return; } - userInfo = body.getJSONObject("user"); - if (Objects.isNull(userInfo)) { + data = body.getJSONObject("user"); + if (Objects.isNull(data)) { return; } - fansCnt[0] = userInfo.getInteger("follower_count"); - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue()); - if (Objects.nonNull(fansCnt[0])) { - if (Objects.nonNull(account)) { - accountDao.updateFans(account.getId(), fansCnt[0], new Date()); - } else { - accountDao.save(Account.builder() - .cookiesStatus(true) - .phoneNo(accountNo) - .type(this.getType().getValue()) - .fansCnt(fansCnt[0]) - .reportDate(new Date()) - .done(false) - .build()); - } - } + reportAccountDto.set(ReportAccountDto.builder() + .fansCnt(data.getInteger("follower_count")) + .followCnt(data.getInteger("following_count")) + .likeCnt(data.getInteger("total_favorited")) + .accountId(data.getString("unique_id")) + .userOtherId(data.getString("uid")) + .accountName(data.getString("nickname")) + .build() + ); + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue()); + if (Objects.nonNull(account)) + accountDao.updateMsg(account.getId(), reportAccountDto.get().getAccountName(), reportAccountDto.get().getFansCnt(), new Date()); + else + accountDao.save(Account.builder() + .cookiesStatus(true) + .accountNo(accountNo) + .type(this.getType().getValue()) + .fansCnt(reportAccountDto.get().getFansCnt()) + .accountName(reportAccountDto.get().getAccountName()) + .reportDate(new Date()) + .done(false) + .build()); } } catch (Exception e) { this.exitBrowser(accountNo, uuid); @@ -718,7 +725,7 @@ public class DouYinCrawl implements CrawlStrategy { } }); this.exitBrowser(accountNo, uuid); - return fansCnt[0]; + return reportAccountDto.get(); } /** diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java index 03874f6..8376f0f 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java @@ -12,6 +12,7 @@ import cn.fw.freya.model.data.FwCookie; import cn.fw.freya.model.data.ResponseReceived; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.service.crawl.CrawlStrategy; import cn.fw.freya.service.data.AccountService; import cn.fw.freya.utils.DateUtil; @@ -30,20 +31,29 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.CookieStore; import org.apache.http.impl.client.BasicCookieStore; +import org.apache.logging.log4j.util.PropertiesUtil; import org.openqa.selenium.By; import org.openqa.selenium.Cookie; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.ui.WebDriverWait; +import org.springframework.context.SmartLifecycle; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.support.PropertiesLoaderUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import javax.annotation.Resource; +import java.io.BufferedWriter; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; import java.math.BigDecimal; import java.math.RoundingMode; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.*; @@ -62,8 +72,9 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor @SuppressWarnings("Duplicates") -public class KuaiShouCrawl implements CrawlStrategy { +public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle { + private boolean isRunning = false; private final VideoPoolDao videoPoolDao; private final LivePoolDao livePoolDao; private final AccountDao accountDao; @@ -89,25 +100,46 @@ public class KuaiShouCrawl implements CrawlStrategy { final List accountList = accountDao.getAllKSAccount(); accountList.forEach(item -> Arrays.stream(DataTypeEnum.values()).forEach(item1 -> - threadPoolExecutor.execute(() -> { - final String accountNo = item.getPhoneNo(); - final Integer typeValue = item1.getValue(); - String key = accountNo + "#" + typeValue; - final String ns_sig3 = this.getNS_sig3(accountNo, typeValue); - if (Objects.nonNull(ns_sig3)) - sig3Map.put(key, ns_sig3); - }) + threadPoolExecutor.execute(() -> this.task(item.getAccountNo(), item1.getValue())) ) ); return true; } + /** + * 获取类型签名task + * + * @param accountNo 账户号 + * @param dataType 数据类型 + */ + public void task(String accountNo, Integer dataType) { + String key = accountNo + "#" + dataType; + final String ns_sig3 = this.getNS_sig3(accountNo, dataType, true); + if (Objects.nonNull(ns_sig3)) + sig3Map.put(key, ns_sig3); + } + + /** + * 停止设置sig3Map线程池 + * + * @return + */ public boolean stopSetSig3Map() { threadPoolExecutor.shutdownNow(); return true; } /** + * 清空sig3Map + * + * @return + */ + public boolean cleanSig3Map() { + sig3Map.clear(); + return true; + } + + /** * 获取sig3Map */ public String getSig3Map() { @@ -117,8 +149,13 @@ public class KuaiShouCrawl implements CrawlStrategy { /** * 设置sig3Map */ - public boolean setMapFromString(String jsonStr) { - HashMap map = JSON.parseObject(JSON.parseObject(jsonStr).getString("jsonStr"), new TypeReference<>() { + public boolean setMapFromString(String jsonStr, boolean interior) { + String jsonString; + if (interior) + jsonString = jsonStr; + else + jsonString = JSON.parseObject(jsonStr).getString("jsonStr"); + HashMap map = JSON.parseObject(jsonString, new TypeReference<>() { }, Feature.OrderedField); map.forEach(sig3Map::put); return true; @@ -215,7 +252,7 @@ public class KuaiShouCrawl implements CrawlStrategy { CookieStore cookieStore = new BasicCookieStore(); cookies.setCookieStore(cookieStore); Date previousDay = DateUtil.getPreviousDay(new Date()); - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue()); + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue(), false); if (Objects.isNull(ns_sig3)) return null; Map params = new LinkedHashMap<>(); @@ -243,8 +280,12 @@ public class KuaiShouCrawl implements CrawlStrategy { if (!StringUtils.hasText(res)) { throw new BusinessException("调用快手[视频]接口失败"); } + if (Objects.equals(response.getInteger("result"), 500002)) { + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.VIDEO.getValue())); + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息"); + } JSONArray videoJsonArray = Optional.ofNullable(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject()).getJSONArray("photoList")).orElse(new JSONArray()); - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue()); + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue()); // 视频数据存库 List videoPoolList = new ArrayList<>(videoJsonArray.size()); videoJsonArray.forEach(item -> { @@ -286,7 +327,7 @@ public class KuaiShouCrawl implements CrawlStrategy { .playCount(Optional.ofNullable(obj.getInteger("playCount")).orElse(0))// .likeCount(Optional.ofNullable(obj.getInteger("likeCount")).orElse(0))// .commentCount(Optional.ofNullable(obj.getInteger("commentCount")).orElse(0))// - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .publishTime(PublicUtil.parseDate(obj.getString("publishTime")))// .videoUrl("https://www.kuaishou.com/short-video/" + obj.getString("workId")) @@ -305,7 +346,7 @@ public class KuaiShouCrawl implements CrawlStrategy { videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 } else { final VideoPool nullVideo = VideoPool.builder() - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) .resourceType(ResourceTypeEnum.VIDEO.getValue()) .reportDate(new Date()) @@ -341,7 +382,7 @@ public class KuaiShouCrawl implements CrawlStrategy { Date endTime = DateUtil.getThisDayMaxTime(previousDay); Date startTime = DateUtil.getThisDayMinTime(previousDay); //Date startTime = DateUtil.getThisDayMinTime(new Date(previousDay.getTime() - 7 * 24 * 3600 * 1000L));// 补数据使用 - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue()); + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue(), false); if (Objects.isNull(ns_sig3)) return null; Map params = new LinkedHashMap<>(); @@ -371,6 +412,10 @@ public class KuaiShouCrawl implements CrawlStrategy { if (!StringUtils.hasText(res)) { throw new BusinessException("调用快手[视频]接口失败"); } + if (Objects.equals(response.getInteger("result"), 500002)) { + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.LIVE.getValue())); + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息"); + } JSONObject dataJSONObject = response.getJSONObject("data"); JSONArray dataJSONArray = dataJSONObject.getJSONArray("details"); /*List collect = new ArrayList<>(); @@ -424,49 +469,9 @@ public class KuaiShouCrawl implements CrawlStrategy { * userHead: "https://tx2.a.kwimgs.com/uhead/AB/2021/11/11/12/BMjAyMTExMTExMjUwMzZfMjU2MTc2NDMyMV8xX2hkMjg5XzMyOA==_s.jpg" * -userName: "长安汽车。小明聊聊车" */ - /*String coverUrl = obj.getString("liveCover");// 直播封面地址 - String playbackUrl = null; - final Double liveDuration = obj.getDouble("liveDuration");// 数据接口返回直播时长 - final Long liveStartTimeStamp = obj.getLong("liveTime");// 数据接口返回开播时间戳 - if (liveDuration >= 60) {// 直播时长大于等于60分钟才记录回放信息 - if (!CollectionUtils.isEmpty(finalCollect)) { - JSONObject playbackMsg; - if (Objects.equals(finalCollect.size(), 1) && Objects.equals(dataJSONArray.size(), 1)) { - playbackMsg = finalCollect.get(0); - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长 - final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳 - final double timeSubAbs = Math.abs(liveDuration - duration); - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); - // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等 - // (liveStartSub < 2)->说明开播时间几乎一样 - if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) { - playbackMsg = null; - log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); - } - } else { - List collect1 = finalCollect.stream().filter(item1 -> { - Double duration = item1.getDouble("duration"); - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳 - final double timeSubAbs = Math.abs(liveDuration - duration); - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue(); - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10; - }).collect(Collectors.toList()); - if (Objects.equals(collect1.size(), 1)) { - playbackMsg = collect1.get(0); - } else { - playbackMsg = null; - log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo)); - } - } - if (Objects.nonNull(playbackMsg)) { - coverUrl = playbackMsg.getString("coverUrl"); - playbackUrl = this.playbackBaseUrl + playbackMsg.getString("productId"); - } - } - }*/ livePoolList.add(LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .commentUserCnt(Optional.ofNullable(obj.getInteger("commentUv")).orElse(0)) .consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) @@ -499,7 +504,7 @@ public class KuaiShouCrawl implements CrawlStrategy { log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的快手直播数据"); final LivePool nullLive = LivePool.builder() .type(this.getType().getValue()) - .phoneNo(accountNo) + .accountNo(accountNo) .reportDate(new Date()) .build(); livePoolDao.save(nullLive); @@ -517,7 +522,7 @@ public class KuaiShouCrawl implements CrawlStrategy { object.put("playbackUrl", null); if (CollectionUtils.isEmpty(collect)) return object; - final String accountNo = dbLive.getPhoneNo(); + final String accountNo = dbLive.getAccountNo(); final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长 final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳 if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息 @@ -565,8 +570,8 @@ public class KuaiShouCrawl implements CrawlStrategy { JSONArray objects = new JSONArray(); if (Objects.isNull(searchKey)) return objects; - Map params = new HashMap<>(); - Map params1 = new HashMap<>(); + Map params = new LinkedHashMap<>(); + Map params1 = new LinkedHashMap<>(); params1.put("principalId", searchKey); params1.put("pcursor", ""); params1.put("count", 150); @@ -610,17 +615,17 @@ public class KuaiShouCrawl implements CrawlStrategy { */ @Override @Transactional - public Integer updateAccountFans(String accountNo) throws IOException { - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); - if (Objects.nonNull(hasFoundFansCnt)) { - //return hasFoundFansCnt; + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException { + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); + if (Objects.nonNull(hasFoundAccountMsg)) { + return hasFoundAccountMsg; } HttpCookies cookies = HttpCookies.custom(); CookieStore cookieStore = new BasicCookieStore(); cookies.setCookieStore(cookieStore); Map params = new HashMap<>(); params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo)); - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue()); + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue(), false); if (Objects.isNull(ns_sig3)) return null; HttpConfig config = HttpConfig.custom() @@ -643,24 +648,33 @@ public class KuaiShouCrawl implements CrawlStrategy { if (!StringUtils.hasText(res)) { throw new BusinessException("调用快手[直播]接口失败"); } - Integer fansNum; - if (Objects.equals(response.getInteger("result"), 500002)) - fansNum = 0; - else - fansNum = response.getJSONObject("data").getInteger("fansCnt"); - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue()); + if (Objects.equals(response.getInteger("result"), 500002)) { + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.FANS.getValue())); + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息"); + } + JSONObject data = response.getJSONObject("data"); + ReportAccountDto reportAccountDto = ReportAccountDto.builder() + .fansCnt(data.getInteger("fansCnt")) + .followCnt(data.getInteger("followCnt")) + .likeCnt(data.getInteger("likeCnt")) + .accountId(data.getString("userId")) + .userOtherId(data.getString("userKwaiId")) + .accountName(data.getString("userName")) + .build(); + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue()); if (Objects.nonNull(account)) - accountDao.updateFans(account.getId(), fansNum, new Date()); + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date()); else accountDao.save(Account.builder() .cookiesStatus(true) - .phoneNo(accountNo) + .accountNo(accountNo) .type(this.getType().getValue()) - .fansCnt(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt")) + .fansCnt(reportAccountDto.getFansCnt()) + .accountName(reportAccountDto.getAccountName()) .reportDate(new Date()) .done(false) .build()); - return Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt"); + return reportAccountDto; } /** @@ -700,12 +714,16 @@ public class KuaiShouCrawl implements CrawlStrategy { * * @param accountNo 快手账户号 * @param type 密钥类型(1:粉丝, 2:短视频, 3:直播) + * @param retryGet 是否重新获取 */ - public String getNS_sig3(String accountNo, Integer type) { + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) { final String key = accountNo + "#" + type; - String NS_sig3 = sig3Map.get(key); - if (StringUtils.hasText(NS_sig3)) - return NS_sig3; + String NS_sig3; + if (!retryGet) { + NS_sig3 = sig3Map.get(key); + if (StringUtils.hasText(NS_sig3)) + return NS_sig3; + } final String uuid = UUID.randomUUID().toString().replace("-", ""); final WebDriver driver = this.getKSDriver(accountNo, uuid); String targetUrl = null; @@ -852,7 +870,7 @@ public class KuaiShouCrawl implements CrawlStrategy { * @return */ public String getRandomUserCookies() { - final List cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getPhoneNo(), this.getType().getValue()); + final List cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getAccountNo(), this.getType().getValue()); StringBuffer sb = new StringBuffer(); sb.append("clientid=3;"); cookies.forEach(item -> { @@ -903,4 +921,78 @@ public class KuaiShouCrawl implements CrawlStrategy { return true; } + @Override + public int getPhase() { + return 1;// 默认为0 + } + + @Override + public boolean isAutoStartup() { + return true;// 默认为false + } + + @Override + public void stop(Runnable callback) { + log.info("springIOC停止, 将签名信息从Map中导出到配置文件"); + callback.run(); + isRunning = false; + final HashMap props = new HashMap<>(); + props.put("KSAccountNS_sig", JSON.toJSONString(sig3Map)); + updateProperties("NS_sig3Msg.properties", props); + } + + @Override + public void start() { + log.info("springIOC启动, 将签名信息从配置文件读出并导入到Map中"); + isRunning = true; + Properties props; + try { + props = PropertiesLoaderUtils.loadAllProperties("NS_sig3Msg.properties"); + for (Object key : props.keySet()) { + if (Objects.equals(key, "KSAccountNS_sig")) + this.setMapFromString(props.get(key).toString(), true); + } + } catch (IOException e) { + log.error(e.getMessage()); + } + } + + @Override + public void stop() { + log.info("stop"); + isRunning = false; + } + + @Override + public boolean isRunning() { + return isRunning; + } + + public void updateProperties(String fileName, Map keyValueMap) { + String filePath = Objects.requireNonNull(PropertiesUtil.class.getClassLoader().getResource(fileName)).getFile(); + Properties props; + BufferedWriter bw = null; + try { + filePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8); + log.debug("updateProperties propertiesPath:" + filePath); + props = PropertiesLoaderUtils.loadProperties(new ClassPathResource(fileName)); + log.debug("updateProperties old:" + props); + bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath)));// 写入属性文件 + //props.clear();// 清空旧的文件 + for (String key : keyValueMap.keySet()) { + props.setProperty(key, keyValueMap.get(key)); + } + log.debug("updateProperties new:" + props); + props.store(bw, ""); + } catch (IOException e) { + log.error(e.getMessage()); + } finally { + try { + assert bw != null; + bw.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } } diff --git a/src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java b/src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java index 4128c37..73d8b3a 100644 --- a/src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java +++ b/src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java @@ -35,13 +35,13 @@ public class AccountRpcService { /** * 账号cookie失效, 上报服务器 * - * @param phoneNo 账号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountNo 账号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) * @return 结果 */ - public boolean pushExpireAccount(String phoneNo, Integer type) { + public boolean pushExpireAccount(String accountNo, Integer type) { Map params = new HashMap<>(); - params.put("account", phoneNo); + params.put("account", accountNo); params.put("type", type); HttpConfig config = HttpConfig.custom() .url(getBaseUrl() + PathConstant.EXPIRE_ACCOUNT) @@ -55,7 +55,7 @@ public class AccountRpcService { String message = Optional.ofNullable(resObj.getString("message")).orElse("账号cookie失效上报失败"); throw new BusinessException(status, message); } - accountService.updateAccountCookiesStatus(phoneNo, type, false); + accountService.updateAccountCookiesStatus(accountNo, type, false); return true; } @@ -86,7 +86,7 @@ public class AccountRpcService { jsonArray.forEach(item -> { final JSONObject obj = (JSONObject) item; final AccountDto dto = new AccountDto(); - dto.setPhoneNo(obj.getString("account")); + dto.setAccountNo(obj.getString("account")); dto.setPlaybackSearchKey(obj.getString("playbackSearchKey")); dto.setGroupId(obj.getLong("groupId")); list.add(dto); diff --git a/src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java b/src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java index 0930aff..86de7f2 100644 --- a/src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java +++ b/src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java @@ -4,6 +4,7 @@ import cn.fw.freya.common.PathConstant; import cn.fw.freya.model.data.pool.LivePool; import cn.fw.freya.model.data.pool.VideoPool; import cn.fw.freya.model.dto.rpc.LivePoolDto; +import cn.fw.freya.model.dto.rpc.ReportAccountDto; import cn.fw.freya.model.dto.rpc.VideoPoolDto; import cn.fw.freya.utils.RequestUtil; import cn.fw.freya.utils.http.HttpConfig; @@ -32,18 +33,19 @@ public class ReportRpcService { private String baseUrl; /** - * 账号粉丝数上报 + * 账户信息上报 * - * @param accountNo 账户号 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) - * @param fans 粉丝数 + * @param accountNo 账户号 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) + * @param accountMsg 账户信息 * @return */ - public boolean reportFansCnt(String accountNo, Integer type, Integer fans) { + public boolean reportAccountMsg(String accountNo, Integer type, ReportAccountDto accountMsg) { Map params = new HashMap<>(); params.put("account", accountNo); params.put("type", type); - params.put("fans", fans); + params.put("fans", accountMsg.getFansCnt()); + params.put("accountName", accountMsg.getAccountName()); HttpConfig config = HttpConfig.custom() .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName()) .url(getBaseUrl() + PathConstant.REPORT_FANS) diff --git a/src/main/java/cn/fw/freya/task/DataCaptureTask.java b/src/main/java/cn/fw/freya/task/DataCaptureTask.java index 54c4b4b..2a28b6b 100644 --- a/src/main/java/cn/fw/freya/task/DataCaptureTask.java +++ b/src/main/java/cn/fw/freya/task/DataCaptureTask.java @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -51,7 +50,7 @@ public class DataCaptureTask { /** * 每分钟执行多线程同时抓取数据 */ - @Scheduled(cron = "0 0/1 * * * ?") + //@Scheduled(cron = "0 0/1 * * * ?") public void capture() { final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); BlockingQueue queue = threadPoolExecutor.getQueue();// 获取工作队列 @@ -78,11 +77,11 @@ public class DataCaptureTask { }); final Integer accountType = currentAccount.getType();// 当前账户类型 final Integer nextAccountType = Optional.ofNullable(nextAccount.getType()).orElse(-1);// 下一个账户类型 - final String nextAccountNo = Optional.ofNullable(nextAccount.getPhoneNo()).orElse("");// 下一个账户号 + final String nextAccountNo = Optional.ofNullable(nextAccount.getAccountNo()).orElse("");// 下一个账户号 final Date todayDayMinTime = DateUtil.getThisDayMinTime(new Date()); if (accountType.equals(2) && nextAccountType.equals(2)) { if (!Objects.equals(nextAccountNo, "") && !Objects.equals(nextAccountType, -1) && - (Objects.nonNull(common.getHasFoundFansCnt(nextAccountNo, 2, todayDayMinTime)) && + (Objects.nonNull(common.getHasFoundAccountMsg(nextAccountNo, 2, todayDayMinTime)) && Objects.nonNull(common.getHasFoundVideo(nextAccountNo, 2, todayDayMinTime)) && Objects.nonNull(common.getHasFoundLive(nextAccountNo, 2, todayDayMinTime)) )) { @@ -97,7 +96,7 @@ public class DataCaptureTask { /** * 每2分钟执行抓取数据 */ - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) public void captureLivePlayback() { Double durationThreshold = 60d;// 设置直播时长阈值 final Random random = new Random(); @@ -107,7 +106,7 @@ public class DataCaptureTask { .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据 Collection> values = withoutPlaybackLive .stream() - .collect(Collectors.groupingBy(LivePool::getPhoneNo)) + .collect(Collectors.groupingBy(LivePool::getAccountNo)) .values() .stream() .peek(item -> item.sort(Comparator.comparing(LivePool::getGetPlaybackFailTimes)))// 把某个人每条直播按失败次数排序 @@ -115,8 +114,8 @@ public class DataCaptureTask { .limit(5)// 一次找5个人的直播 .collect(Collectors.toList()); for (List list : values) {// 遍历每个人的直播集合 - final String accountNo = list.get(0).getPhoneNo(); - Account account = accountDao.findByPhoneNoAndType(accountNo, 1);// 获取账号实体 + final String accountNo = list.get(0).getAccountNo(); + Account account = accountDao.findByAccountNoAndType(accountNo, 1);// 获取账号实体 final List playbackMsg = this.getPlaybackMsg(account);// 获取该人的直播回放信息 boolean flag = false; for (LivePool item : list) { @@ -170,7 +169,7 @@ public class DataCaptureTask { if (Objects.nonNull(playbackSearchKey)) userLivePlayback = kuaiShouCrawl.getUserLivePlayback(playbackSearchKey); else - userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getPhoneNo()); + userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getAccountNo()); final List collect = userLivePlayback .stream() .filter(item -> { diff --git a/src/main/java/cn/fw/freya/utils/RequestUtil.java b/src/main/java/cn/fw/freya/utils/RequestUtil.java index 6e423b8..c9133d7 100644 --- a/src/main/java/cn/fw/freya/utils/RequestUtil.java +++ b/src/main/java/cn/fw/freya/utils/RequestUtil.java @@ -696,10 +696,7 @@ public class RequestUtil { // 设置通用的请求属性 conn.setRequestProperty("accept", "*/*"); conn.setRequestProperty("connection", "Keep-Alive"); - conn.setRequestProperty("Host", "cp.kuaishou.com"); conn.setRequestProperty("Content-Type", "application/json"); - conn.setRequestProperty("Content-Length", "84"); - conn.setRequestProperty("Cookie", "kuaishou.web.cp.api_ph=c634819dfa9a1da1762bafcf5d195fd101b4;kuaishou.web.cp.api_st=ChZrdWFpc2hvdS53ZWIuY3AuYXBpLnN0EqAB_xuzZVzMjD9DNrreTtCT1cQDRKvB60dNkgWBOQI3mk8_F0WAM1qRKnUtcZYuGH0JTMUntF7JMrs8L7vYLP9tpQ25ppobabbfAcPBqztI7OJdc7MqSwrjanYe6PRZCHnU-tALse-CVxzMFtCl304ekWU1o9fjm7VBQ9EEkC9Q35erQ2RbrnucqQI5E3p_9cd0JE9J8odcLgXOjbWQvT0OuhoSPRRvGtXySX7EnaufBy__uLw_IiA-3ieQkuU_bNGof-EIcBSlmCzad9Ut9hWRS5nEILA9FCgFMAE;_did=web_656080669F3F2217;userId=2079628287;did=web_94f084024029be79aade1f079e2b82e1731a;"); // 发送POST请求必须设置如下两行 conn.setDoOutput(true); conn.setDoInput(true); diff --git a/src/main/resources/NS_sig3Msg.properties b/src/main/resources/NS_sig3Msg.properties new file mode 100644 index 0000000..63dae6b --- /dev/null +++ b/src/main/resources/NS_sig3Msg.properties @@ -0,0 +1,3 @@ +# +#Thu Jan 27 15:59:12 CST 2022 +KSAccountNS_sig={} diff --git a/src/main/resources/alert.sql b/src/main/resources/alert.sql new file mode 100644 index 0000000..79d423b --- /dev/null +++ b/src/main/resources/alert.sql @@ -0,0 +1,19 @@ +alter table ACCOUNT drop constraint UKB04BR4OH2RS73RNUSEXU2NLWG; +alter table ACCOUNT drop constraint UKSOWL8PT6SDTPQPVTXFDP0NPU0; +alter table ACCOUNT drop column ACCOUNT_NO; +alter table ACCOUNT alter column PHONE_NO rename to ACCOUNT_NO; + +drop index IDXTGFKFY1N98UJQVJ5ME0VR3FSV; +drop index IDXKWOBILLHNTYYT02FVAQAM5G5; +alter table COOKIE drop column ACCOUNT_NO; +alter table COOKIE alter column PHONE_NO rename to ACCOUNT_NO; + +alter table LIVE_POOL drop constraint UKANITO7VXALLS54057JU6KJHDF; +alter table LIVE_POOL drop constraint UKRQ8QS6Q2H9NPW3GV40B86LRXM; +alter table LIVE_POOL drop column ACCOUNT_NO; +alter table LIVE_POOL alter column PHONE_NO rename to ACCOUNT_NO; + +alter table VIDEO_POOL drop constraint UKL4792GABX9U74HBTHQ0VQM1FG; +alter table VIDEO_POOL drop constraint UKQ3ROPUUMLOIHN45FGCSUK77G; +alter table VIDEO_POOL drop column ACCOUNT_NO; +alter table VIDEO_POOL alter column PHONE_NO rename to ACCOUNT_NO; -- libgit2 0.22.2