third / fw-freya

Commit d472478d333c0cbbca7312129a8834d36de1962c

Authored by 王明元
1 parent e17d2707

2022年6月30日18:19:28 调用接口时候发现500002错误码, 直接更新sig信息

Inline Side-by-side
Showing 31 changed files with 520 additions and 315 deletions
src/main/java/cn/fw/freya/controller/BLBLController.java
  View file @d472478
@@ -2,6 +2,7 @@ package cn.fw.freya.controller; @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2
3 import cn.fw.freya.model.data.pool.LivePool; 3 import cn.fw.freya.model.data.pool.LivePool;
4 import cn.fw.freya.model.data.pool.VideoPool; 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 import cn.fw.freya.service.crawl.impl.BilibiliCrawl; 6 import cn.fw.freya.service.crawl.impl.BilibiliCrawl;
6 import lombok.RequiredArgsConstructor; 7 import lombok.RequiredArgsConstructor;
7 import org.springframework.validation.annotation.Validated; 8 import org.springframework.validation.annotation.Validated;
@@ -52,8 +53,8 @@ public class BLBLController { @@ -52,8 +53,8 @@ public class BLBLController {
52 * @param accountNo 账户号 53 * @param accountNo 账户号
53 */ 54 */
54 @GetMapping("/updateAccountFans") 55 @GetMapping("/updateAccountFans")
55 - public Integer updateAccountFans(String accountNo) throws IOException {  
56 - return bilibiliCrawl.updateAccountFans(accountNo); 56 + public ReportAccountDto updateAccountFans(String accountNo) throws IOException {
  57 + return bilibiliCrawl.updateAccountMsg(accountNo);
57 } 58 }
58 59
59 /** 60 /**
src/main/java/cn/fw/freya/controller/DCDController.java
  View file @d472478
@@ -2,6 +2,7 @@ package cn.fw.freya.controller; @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2
3 import cn.fw.freya.model.data.pool.LivePool; 3 import cn.fw.freya.model.data.pool.LivePool;
4 import cn.fw.freya.model.data.pool.VideoPool; 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 import cn.fw.freya.service.crawl.impl.DongCheDiCrawl; 6 import cn.fw.freya.service.crawl.impl.DongCheDiCrawl;
6 import lombok.RequiredArgsConstructor; 7 import lombok.RequiredArgsConstructor;
7 import org.springframework.validation.annotation.Validated; 8 import org.springframework.validation.annotation.Validated;
@@ -61,8 +62,8 @@ public class DCDController { @@ -61,8 +62,8 @@ public class DCDController {
61 * @param accountNo 账户号 62 * @param accountNo 账户号
62 */ 63 */
63 @GetMapping("/updateAccountFans") 64 @GetMapping("/updateAccountFans")
64 - public Integer updateAccountFans(String accountNo) {  
65 - return dongCheDiCrawl.updateAccountFans(accountNo); 65 + public ReportAccountDto updateAccountFans(String accountNo) {
  66 + return dongCheDiCrawl.updateAccountMsg(accountNo);
66 } 67 }
67 68
68 /** 69 /**
src/main/java/cn/fw/freya/controller/DYController.java
  View file @d472478
@@ -3,6 +3,7 @@ package cn.fw.freya.controller; @@ -3,6 +3,7 @@ package cn.fw.freya.controller;
3 import cn.fw.freya.model.data.LiveOverview; 3 import cn.fw.freya.model.data.LiveOverview;
4 import cn.fw.freya.model.data.pool.LivePool; 4 import cn.fw.freya.model.data.pool.LivePool;
5 import cn.fw.freya.model.data.pool.VideoPool; 5 import cn.fw.freya.model.data.pool.VideoPool;
  6 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
6 import cn.fw.freya.service.crawl.impl.DouYinCrawl; 7 import cn.fw.freya.service.crawl.impl.DouYinCrawl;
7 import lombok.RequiredArgsConstructor; 8 import lombok.RequiredArgsConstructor;
8 import org.springframework.validation.annotation.Validated; 9 import org.springframework.validation.annotation.Validated;
@@ -62,8 +63,8 @@ public class DYController { @@ -62,8 +63,8 @@ public class DYController {
62 * @param accountNo 账户号 63 * @param accountNo 账户号
63 */ 64 */
64 @GetMapping("/updateAccountFans") 65 @GetMapping("/updateAccountFans")
65 - public Integer updateAccountFans(String accountNo) {  
66 - return douyinCrawl.updateAccountFans(accountNo); 66 + public ReportAccountDto updateAccountFans(String accountNo) {
  67 + return douyinCrawl.updateAccountMsg(accountNo);
67 } 68 }
68 69
69 /** 70 /**
src/main/java/cn/fw/freya/controller/KSController.java
  View file @d472478
@@ -2,6 +2,7 @@ package cn.fw.freya.controller; @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2
3 import cn.fw.freya.model.data.pool.LivePool; 3 import cn.fw.freya.model.data.pool.LivePool;
4 import cn.fw.freya.model.data.pool.VideoPool; 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 import cn.fw.freya.service.crawl.impl.KuaiShouCrawl; 6 import cn.fw.freya.service.crawl.impl.KuaiShouCrawl;
6 import com.alibaba.fastjson.JSONObject; 7 import com.alibaba.fastjson.JSONObject;
7 import lombok.RequiredArgsConstructor; 8 import lombok.RequiredArgsConstructor;
@@ -59,8 +60,8 @@ public class KSController { @@ -59,8 +60,8 @@ public class KSController {
59 * @param accountNo 账户号 60 * @param accountNo 账户号
60 */ 61 */
61 @GetMapping("/updateAccountFans") 62 @GetMapping("/updateAccountFans")
62 - public Integer updateAccountFans(String accountNo) throws IOException {  
63 - return kuaishouCrawl.updateAccountFans(accountNo); 63 + public ReportAccountDto updateAccountFans(String accountNo) throws IOException {
  64 + return kuaishouCrawl.updateAccountMsg(accountNo);
64 } 65 }
65 66
66 /** 67 /**
@@ -79,13 +80,12 @@ public class KSController { @@ -79,13 +80,12 @@ public class KSController {
79 * @param accountNo 账户号 80 * @param accountNo 账户号
80 */ 81 */
81 @GetMapping("/getNS_sig3") 82 @GetMapping("/getNS_sig3")
82 - public String getNS_sig3(String accountNo, Integer type) {  
83 - return kuaishouCrawl.getNS_sig3(accountNo, type); 83 + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) {
  84 + return kuaishouCrawl.getNS_sig3(accountNo, type, retryGet);
84 } 85 }
85 86
86 /** 87 /**
87 * 设置setSig3Map 88 * 设置setSig3Map
88 - *  
89 */ 89 */
90 @GetMapping("/setSig3Map") 90 @GetMapping("/setSig3Map")
91 public boolean setSig3Map() { 91 public boolean setSig3Map() {
@@ -94,16 +94,14 @@ public class KSController { @@ -94,16 +94,14 @@ public class KSController {
94 94
95 /** 95 /**
96 * 设置setSig3Map 96 * 设置setSig3Map
97 - *  
98 */ 97 */
99 @PostMapping("/setMapFromString") 98 @PostMapping("/setMapFromString")
100 public boolean setMapFromString(@RequestBody String jsonStr) { 99 public boolean setMapFromString(@RequestBody String jsonStr) {
101 - return kuaishouCrawl.setMapFromString(jsonStr); 100 + return kuaishouCrawl.setMapFromString(jsonStr, false);
102 } 101 }
103 102
104 /** 103 /**
105 * 停止设置setSig3Map 104 * 停止设置setSig3Map
106 - *  
107 */ 105 */
108 @GetMapping("/stopSetSig3Map") 106 @GetMapping("/stopSetSig3Map")
109 public boolean stopSetSig3Map() { 107 public boolean stopSetSig3Map() {
@@ -111,8 +109,15 @@ public class KSController { @@ -111,8 +109,15 @@ public class KSController {
111 } 109 }
112 110
113 /** 111 /**
  112 + * 清空Sig3Map
  113 + */
  114 + @GetMapping("/cleanSig3Map")
  115 + public boolean cleanSig3Map() {
  116 + return kuaishouCrawl.cleanSig3Map();
  117 + }
  118 +
  119 + /**
114 * 获取getSig3Map 120 * 获取getSig3Map
115 - *  
116 */ 121 */
117 @GetMapping("/getSig3Map") 122 @GetMapping("/getSig3Map")
118 public String getSig3Map() { 123 public String getSig3Map() {
src/main/java/cn/fw/freya/controller/OtherController.java
  View file @d472478
@@ -5,6 +5,7 @@ import cn.fw.freya.model.data.Account; @@ -5,6 +5,7 @@ import cn.fw.freya.model.data.Account;
5 import cn.fw.freya.model.data.pool.LivePool; 5 import cn.fw.freya.model.data.pool.LivePool;
6 import cn.fw.freya.model.dto.other.ReportLiveDto; 6 import cn.fw.freya.model.dto.other.ReportLiveDto;
7 import cn.fw.freya.model.dto.other.ReportVideoDto; 7 import cn.fw.freya.model.dto.other.ReportVideoDto;
  8 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
8 import cn.fw.freya.service.CommonBizService; 9 import cn.fw.freya.service.CommonBizService;
9 import cn.fw.freya.service.crawl.impl.Common; 10 import cn.fw.freya.service.crawl.impl.Common;
10 import cn.fw.freya.service.rpc.AccountRpcService; 11 import cn.fw.freya.service.rpc.AccountRpcService;
@@ -86,16 +87,16 @@ public class OtherController { @@ -86,16 +87,16 @@ public class OtherController {
86 } 87 }
87 88
88 /** 89 /**
89 - * 上报账户粉丝数 90 + * 上报账户信息
90 * 91 *
91 - * @param account 账号  
92 - * @param type 账号类型  
93 - * @param fansCnt 粉丝数 92 + * @param account 账号
  93 + * @param type 账号类型
  94 + * @param accountMsg 账户信息
94 * @return 95 * @return
95 */ 96 */
96 - @PostMapping("/reportFansCnt")  
97 - public ResponseMessage<Boolean> reportFansCnt(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "粉丝数量不能为空") Integer fansCnt) {  
98 - return ResponseMessage.success(reportRpcService.reportFansCnt(account, type, fansCnt)); 97 + @PostMapping("/reportAccountMsg")
  98 + public ResponseMessage<Boolean> reportAccountMsg(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "账户信息不能为空") ReportAccountDto accountMsg) {
  99 + return ResponseMessage.success(reportRpcService.reportAccountMsg(account, type, accountMsg));
99 } 100 }
100 101
101 /** 102 /**
@@ -175,7 +176,7 @@ public class OtherController { @@ -175,7 +176,7 @@ public class OtherController {
175 * @return 176 * @return
176 */ 177 */
177 @GetMapping("/getWithoutPlaybackLive") 178 @GetMapping("/getWithoutPlaybackLive")
178 - public List<LivePool> getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空")Double durationThreshold) { 179 + public List<LivePool> getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空") Double durationThreshold) {
179 return common.getWithoutPlaybackLive(type, durationThreshold); 180 return common.getWithoutPlaybackLive(type, durationThreshold);
180 } 181 }
181 182
src/main/java/cn/fw/freya/dao/AccountDao.java
  View file @d472478
@@ -18,23 +18,23 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; { @@ -18,23 +18,23 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; {
18 /** 18 /**
19 * 删除指定账号 19 * 删除指定账号
20 * 20 *
21 - * @param phoneNo 21 + * @param accountNo
22 * @param value 22 * @param value
23 */ 23 */
24 @Transactional(rollbackFor = Exception.class) 24 @Transactional(rollbackFor = Exception.class)
25 @Modifying 25 @Modifying
26 - @Query("delete from Account a where a.phoneNo = ?1 and a.type = ?2")  
27 - void deleteByPhoneNoAndType(String phoneNo, Integer value); 26 + @Query("delete from Account a where a.accountNo = ?1 and a.type = ?2")
  27 + void deleteByAccountNoAndType(String accountNo, Integer value);
28 28
29 /** 29 /**
30 * 查询账号 30 * 查询账号
31 * 31 *
32 - * @param phoneNo 32 + * @param accountNo
33 * @param type 33 * @param type
34 * @return 34 * @return
35 */ 35 */
36 - @Query("select a from Account a where a.phoneNo = ?1 and a.type = ?2 ")  
37 - Account findByPhoneNoAndType(String phoneNo, Integer type); 36 + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 ")
  37 + Account findByAccountNoAndType(String accountNo, Integer type);
38 38
39 /** 39 /**
40 * 随机获取一个账号 40 * 随机获取一个账号
@@ -72,45 +72,47 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; { @@ -72,45 +72,47 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; {
72 List<Account> findNotUseAccount(); 72 List<Account> findNotUseAccount();
73 73
74 /** 74 /**
75 - * 更新账户粉丝数 75 + * 更新账户信息
76 * 76 *
77 - * @param id  
78 - * @param fansCnt 77 + * @param id 账户id
  78 + * @param accountName 账户名
  79 + * @param fansCnt 账户粉丝数
79 * @return 80 * @return
80 */ 81 */
81 @Transactional(rollbackFor = Exception.class) 82 @Transactional(rollbackFor = Exception.class)
82 @Modifying 83 @Modifying
83 - @Query("update Account a set a.fansCnt = ?2, a.reportDate = ?3 where a.id = ?1")  
84 - int updateFans(Long id, int fansCnt, Date reportDate); 84 + @Query("update Account a set a.accountName = ?2, a.fansCnt = ?3, a.reportDate = ?4 where a.id = ?1")
  85 + int updateMsg(Long id, String accountName, int fansCnt, Date reportDate);
85 86
86 /** 87 /**
87 * 获取今天上报的粉丝数据 88 * 获取今天上报的粉丝数据
88 * 89 *
89 - * @param phoneNo 账户号 90 + * @param accountNo 账户号
90 * @param type 账户类型 91 * @param type 账户类型
91 * @param reportDate 上报日期 92 * @param reportDate 上报日期
92 * @return 93 * @return
93 */ 94 */
94 - @Query("select a.fansCnt from Account a where a.phoneNo = ?1 and a.type = ?2 and a.reportDate >= ?3")  
95 - Integer getHasReportDate(String phoneNo, Integer type, Date reportDate); 95 + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 and a.reportDate >= ?3")
  96 + List<Account> getHasReportDate(String accountNo, Integer type, Date reportDate);
96 97
97 /** 98 /**
98 * 更新账户cookies状态 99 * 更新账户cookies状态
99 * 100 *
100 - * @param phoneNo 101 + * @param accountNo
101 * @param type 102 * @param type
102 * @param cookiesStatus 103 * @param cookiesStatus
103 */ 104 */
104 @Transactional(rollbackFor = Exception.class) 105 @Transactional(rollbackFor = Exception.class)
105 @Modifying 106 @Modifying
106 - @Query("update Account a set a.cookiesStatus = ?3 where a.phoneNo = ?1 and a.type = ?2")  
107 - void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus); 107 + @Query("update Account a set a.cookiesStatus = ?3 where a.accountNo = ?1 and a.type = ?2")
  108 + void updateAccountCookiesStatus(String accountNo, Integer type, boolean cookiesStatus);
108 109
109 /** 110 /**
110 * 设置账户状态为未完成 111 * 设置账户状态为未完成
  112 + *
111 * @param accountNo 113 * @param accountNo
112 */ 114 */
113 @Modifying 115 @Modifying
114 - @Query("update Account a set a.done = false where a.phoneNo = ?1") 116 + @Query("update Account a set a.done = false where a.accountNo = ?1")
115 void setAccountUndone(String accountNo); 117 void setAccountUndone(String accountNo);
116 } 118 }
src/main/java/cn/fw/freya/dao/CookieDao.java
  View file @d472478
@@ -17,21 +17,21 @@ public interface CookieDao extends JpaRepository&lt;FwCookie, Long&gt; { @@ -17,21 +17,21 @@ public interface CookieDao extends JpaRepository&lt;FwCookie, Long&gt; {
17 /** 17 /**
18 * 删除账号对应的cookie 18 * 删除账号对应的cookie
19 * 19 *
20 - * @param phoneNo 账户号  
21 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 20 + * @param accountNo 账户号
  21 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
22 */ 22 */
23 @Transactional 23 @Transactional
24 @Modifying 24 @Modifying
25 - @Query("delete from FwCookie f where f.phoneNo = ?1 and f.type = ?2")  
26 - void deleteByPhoneNoAndType(String phoneNo, Integer type); 25 + @Query("delete from FwCookie f where f.accountNo = ?1 and f.type = ?2")
  26 + void deleteByAccountNoAndType(String accountNo, Integer type);
27 27
28 /** 28 /**
29 * 查询账号对应平台的cookie 29 * 查询账号对应平台的cookie
30 * 30 *
31 - * @param phoneNo 账户号  
32 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 31 + * @param accountNo 账户号
  32 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
33 * @return 33 * @return
34 */ 34 */
35 - @Query("select f from FwCookie f where f.phoneNo = ?1 and f.type = ?2")  
36 - List<FwCookie> findByPhoneNoAndType(String phoneNo, Integer type); 35 + @Query("select f from FwCookie f where f.accountNo = ?1 and f.type = ?2")
  36 + List<FwCookie> findByAccountNoAndType(String accountNo, Integer type);
37 } 37 }
src/main/java/cn/fw/freya/dao/LivePoolDao.java
  View file @d472478
@@ -17,13 +17,13 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; { @@ -17,13 +17,13 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; {
17 /** 17 /**
18 * 获取今天上报的数据 18 * 获取今天上报的数据
19 * 19 *
20 - * @param phoneNo 账户号 20 + * @param accountNo 账户号
21 * @param type 账户类型 21 * @param type 账户类型
22 * @param reportDate 上报日期 22 * @param reportDate 上报日期
23 * @return 23 * @return
24 */ 24 */
25 - @Query("select live from LivePool live where live.phoneNo = ?1 and live.type = ?2 and live.reportDate >= ?3")  
26 - List<LivePool> getHasReportDate(String phoneNo, Integer type, Date reportDate); 25 + @Query("select live from LivePool live where live.accountNo = ?1 and live.type = ?2 and live.reportDate >= ?3")
  26 + List<LivePool> getHasReportData(String accountNo, Integer type, Date reportDate);
27 27
28 /** 28 /**
29 * 获取没有拿到回播的直播 29 * 获取没有拿到回播的直播
@@ -41,6 +41,6 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; { @@ -41,6 +41,6 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; {
41 * @param accountNo 账户号 41 * @param accountNo 账户号
42 * @return 42 * @return
43 */ 43 */
44 - @Query("select live from LivePool live where live.phoneNo = ?1") 44 + @Query("select live from LivePool live where live.accountNo = ?1")
45 List<LivePool> getLiveByAccountNo(String accountNo); 45 List<LivePool> getLiveByAccountNo(String accountNo);
46 } 46 }
src/main/java/cn/fw/freya/dao/VideoPoolDao.java
  View file @d472478
@@ -21,24 +21,24 @@ public interface VideoPoolDao extends JpaRepository&lt;VideoPool, Long&gt; { @@ -21,24 +21,24 @@ public interface VideoPoolDao extends JpaRepository&lt;VideoPool, Long&gt; {
21 /** 21 /**
22 * 删除 22 * 删除
23 * 23 *
24 - * @param phoneNo 账户号 24 + * @param accountNo 账户号
25 * @param previousDay 前一天 25 * @param previousDay 前一天
26 * @param type 账户类型 26 * @param type 账户类型
27 * @param resourceType 资源类型 27 * @param resourceType 资源类型
28 */ 28 */
29 @Transactional(rollbackFor = Exception.class) 29 @Transactional(rollbackFor = Exception.class)
30 @Modifying 30 @Modifying
31 - @Query("delete from VideoPool where phoneNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4")  
32 - void deleteByPhoneNoAndDate(String phoneNo, Date previousDay, Integer type, Integer resourceType); 31 + @Query("delete from VideoPool where accountNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4")
  32 + void deleteByAccountNoAndDate(String accountNo, Date previousDay, Integer type, Integer resourceType);
33 33
34 /** 34 /**
35 * 获取今天上报的数据 35 * 获取今天上报的数据
36 * 36 *
37 - * @param phoneNo 账户号 37 + * @param accountNo 账户号
38 * @param type 账户类型 38 * @param type 账户类型
39 * @param reportDate 上报日期 39 * @param reportDate 上报日期
40 * @return 40 * @return
41 */ 41 */
42 - @Query("select v from VideoPool v where v.phoneNo = ?1 and v.type = ?2 and v.reportDate >= ?3")  
43 - List<VideoPool> getHasReportDate(String phoneNo, Integer type, Date reportDate); 42 + @Query("select v from VideoPool v where v.accountNo = ?1 and v.type = ?2 and v.reportDate >= ?3")
  43 + List<VideoPool> getHasReportDate(String accountNo, Integer type, Date reportDate);
44 } 44 }
src/main/java/cn/fw/freya/enums/AccountTypeEnum.java
  View file @d472478
@@ -11,7 +11,7 @@ public enum AccountTypeEnum { @@ -11,7 +11,7 @@ public enum AccountTypeEnum {
11 KS(1, "快手"),// 快手 11 KS(1, "快手"),// 快手
12 DY(2, "抖音"),// 抖音 12 DY(2, "抖音"),// 抖音
13 DCD(3, "懂车帝"),// 懂车帝 13 DCD(3, "懂车帝"),// 懂车帝
14 - BILIBILI(4, "哔哩哔哩动画"),// 哔哩哔哩动画 14 + BILIBILI(4, "哔哩哔哩"),// 哔哩哔哩
15 ; 15 ;
16 @Getter 16 @Getter
17 private final Integer value; 17 private final Integer value;
src/main/java/cn/fw/freya/model/data/Account.java
  View file @d472478
@@ -18,16 +18,12 @@ import java.util.Objects; @@ -18,16 +18,12 @@ import java.util.Objects;
18 @AllArgsConstructor 18 @AllArgsConstructor
19 @Builder(toBuilder = true) 19 @Builder(toBuilder = true)
20 @Entity 20 @Entity
21 -@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "type"})) 21 +@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type"}))
22 public class Account { 22 public class Account {
23 @Id 23 @Id
24 @GeneratedValue(strategy = GenerationType.IDENTITY) 24 @GeneratedValue(strategy = GenerationType.IDENTITY)
25 private Long id; 25 private Long id;
26 /** 26 /**
27 - * 账户号  
28 - */  
29 - private String phoneNo;  
30 - /**  
31 * 回播搜索关键词 27 * 回播搜索关键词
32 */ 28 */
33 private String playbackSearchKey; 29 private String playbackSearchKey;
@@ -36,6 +32,10 @@ public class Account { @@ -36,6 +32,10 @@ public class Account {
36 */ 32 */
37 private String accountNo; 33 private String accountNo;
38 /** 34 /**
  35 + * 账户名
  36 + */
  37 + private String accountName;
  38 + /**
39 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 39 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
40 */ 40 */
41 private Integer type; 41 private Integer type;
src/main/java/cn/fw/freya/model/data/FwCookie.java
  View file @d472478
@@ -23,7 +23,7 @@ import java.util.Objects; @@ -23,7 +23,7 @@ import java.util.Objects;
23 @ToString 23 @ToString
24 @AllArgsConstructor 24 @AllArgsConstructor
25 @NoArgsConstructor 25 @NoArgsConstructor
26 -@Table(name = "cookie", indexes = {@Index(columnList = "phoneNo")}) 26 +@Table(name = "cookie", indexes = {@Index(columnList = "accountNo")})
27 public class FwCookie { 27 public class FwCookie {
28 @Id 28 @Id
29 @GeneratedValue(strategy = GenerationType.IDENTITY) 29 @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -31,7 +31,7 @@ public class FwCookie { @@ -31,7 +31,7 @@ public class FwCookie {
31 /** 31 /**
32 * 账户号 32 * 账户号
33 */ 33 */
34 - private String phoneNo; 34 + private String accountNo;
35 /** 35 /**
36 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 36 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
37 */ 37 */
@@ -104,13 +104,13 @@ public class FwCookie { @@ -104,13 +104,13 @@ public class FwCookie {
104 /** 104 /**
105 * WebDriver获取到的cookies->DB 105 * WebDriver获取到的cookies->DB
106 * 106 *
107 - * @param cookie WebDriver获取到的cookie  
108 - * @param phoneNo 账户号 107 + * @param cookie WebDriver获取到的cookie
  108 + * @param accountNo 账户号
109 * @return cookies 109 * @return cookies
110 */ 110 */
111 - public static FwCookie toDb(Cookie cookie, String phoneNo, Integer type) { 111 + public static FwCookie toDb(Cookie cookie, String accountNo, Integer type) {
112 FwCookie fwCookie = new FwCookie(); 112 FwCookie fwCookie = new FwCookie();
113 - fwCookie.setPhoneNo(phoneNo); 113 + fwCookie.setAccountNo(accountNo);
114 fwCookie.setType(type); 114 fwCookie.setType(type);
115 fwCookie.setDomain(cookie.getDomain()); 115 fwCookie.setDomain(cookie.getDomain());
116 fwCookie.setExpiryDate(DateUtil.date2LocalDateTime(cookie.getExpiry())); 116 fwCookie.setExpiryDate(DateUtil.date2LocalDateTime(cookie.getExpiry()));
src/main/java/cn/fw/freya/model/data/LiveOverview.java
  View file @d472478
@@ -18,7 +18,7 @@ import java.util.Date; @@ -18,7 +18,7 @@ import java.util.Date;
18 @Builder(toBuilder = true) 18 @Builder(toBuilder = true)
19 @Entity 19 @Entity
20 @ToString 20 @ToString
21 -@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "reportDate", "type"})) 21 +@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "reportDate", "type"}))
22 public class LiveOverview { 22 public class LiveOverview {
23 @Id 23 @Id
24 @GeneratedValue(strategy = GenerationType.IDENTITY) 24 @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -41,7 +41,7 @@ public class LiveOverview { @@ -41,7 +41,7 @@ public class LiveOverview {
41 /** 41 /**
42 * 账户号 42 * 账户号
43 */ 43 */
44 - private String phoneNo; 44 + private String accountNo;
45 /** 45 /**
46 * 直播场次 46 * 直播场次
47 */ 47 */
src/main/java/cn/fw/freya/model/data/pool/LivePool.java
  View file @d472478
@@ -20,7 +20,7 @@ import java.util.Objects; @@ -20,7 +20,7 @@ import java.util.Objects;
20 @AllArgsConstructor 20 @AllArgsConstructor
21 @Builder(toBuilder = true) 21 @Builder(toBuilder = true)
22 @Entity 22 @Entity
23 -@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "phoneNo", "roomId"})) 23 +@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "accountNo", "roomId"}))
24 public class LivePool { 24 public class LivePool {
25 /** 25 /**
26 * 主键id 26 * 主键id
@@ -35,7 +35,7 @@ public class LivePool { @@ -35,7 +35,7 @@ public class LivePool {
35 /** 35 /**
36 * 账户号 36 * 账户号
37 */ 37 */
38 - private String phoneNo; 38 + private String accountNo;
39 /** 39 /**
40 * 数据上报日期 40 * 数据上报日期
41 */ 41 */
src/main/java/cn/fw/freya/model/data/pool/VideoPool.java
  View file @d472478
@@ -19,7 +19,7 @@ import java.util.Date; @@ -19,7 +19,7 @@ import java.util.Date;
19 @Builder(toBuilder = true) 19 @Builder(toBuilder = true)
20 @Entity 20 @Entity
21 @ToString 21 @ToString
22 -@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "videoId", "type", "reportDate"})) 22 +@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "videoId", "type", "reportDate"}))
23 public class VideoPool { 23 public class VideoPool {
24 @Id 24 @Id
25 @GeneratedValue(strategy = GenerationType.IDENTITY) 25 @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -27,7 +27,7 @@ public class VideoPool { @@ -27,7 +27,7 @@ public class VideoPool {
27 /** 27 /**
28 * 账户号 28 * 账户号
29 */ 29 */
30 - private String phoneNo; 30 + private String accountNo;
31 /** 31 /**
32 * 视频id 32 * 视频id
33 **/ 33 **/
src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java
  View file @d472478
@@ -12,7 +12,7 @@ public class AccountDto { @@ -12,7 +12,7 @@ public class AccountDto {
12 /** 12 /**
13 * 账号 13 * 账号
14 */ 14 */
15 - private String phoneNo; 15 + private String accountNo;
16 /** 16 /**
17 * 回播搜索关键词 17 * 回播搜索关键词
18 */ 18 */
src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java 0 → 100644
  View file @d472478
  1 +package cn.fw.freya.model.dto.rpc;
  2 +
  3 +import lombok.AllArgsConstructor;
  4 +import lombok.Builder;
  5 +import lombok.Data;
  6 +import lombok.NoArgsConstructor;
  7 +
  8 +/**
  9 + * @author unknown
  10 + * @version 1.0
  11 + * @date 2022/6/29 16:10
  12 + * @Description
  13 + */
  14 +@Data
  15 +@NoArgsConstructor
  16 +@AllArgsConstructor
  17 +@Builder(toBuilder = true)
  18 +public class ReportAccountDto {
  19 + /**
  20 + * 粉丝数
  21 + */
  22 + private Integer fansCnt;
  23 + /**
  24 + * 关注数
  25 + */
  26 + private Integer followCnt;
  27 + /**
  28 + * 点赞数
  29 + */
  30 + private Integer likeCnt;
  31 + /**
  32 + * 用户在平台的其他信息id
  33 + */
  34 + private String userOtherId;
  35 + /**
  36 + * 用户id
  37 + */
  38 + private String accountId;
  39 + /**
  40 + * 用户名
  41 + */
  42 + private String accountName;
  43 +}
src/main/java/cn/fw/freya/service/CommonBizService.java
  View file @d472478
@@ -113,7 +113,7 @@ public class CommonBizService { @@ -113,7 +113,7 @@ public class CommonBizService {
113 } 113 }
114 accountList.addAll(list.stream() 114 accountList.addAll(list.stream()
115 .map(item -> Account.builder() 115 .map(item -> Account.builder()
116 - .phoneNo(item.getPhoneNo()) 116 + .accountNo(item.getAccountNo())
117 .playbackSearchKey(item.getPlaybackSearchKey()) 117 .playbackSearchKey(item.getPlaybackSearchKey())
118 .type(typeEnum.getValue()) 118 .type(typeEnum.getValue())
119 .cookiesStatus(Boolean.TRUE) 119 .cookiesStatus(Boolean.TRUE)
src/main/java/cn/fw/freya/service/CrawlBizService.java
  View file @d472478
@@ -4,6 +4,7 @@ import cn.fw.freya.enums.AccountTypeEnum; @@ -4,6 +4,7 @@ import cn.fw.freya.enums.AccountTypeEnum;
4 import cn.fw.freya.model.data.Account; 4 import cn.fw.freya.model.data.Account;
5 import cn.fw.freya.model.data.pool.LivePool; 5 import cn.fw.freya.model.data.pool.LivePool;
6 import cn.fw.freya.model.data.pool.VideoPool; 6 import cn.fw.freya.model.data.pool.VideoPool;
  7 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
7 import cn.fw.freya.service.crawl.CrawlStrategy; 8 import cn.fw.freya.service.crawl.CrawlStrategy;
8 import cn.fw.freya.service.data.AccountService; 9 import cn.fw.freya.service.data.AccountService;
9 import cn.fw.freya.service.rpc.AccountRpcService; 10 import cn.fw.freya.service.rpc.AccountRpcService;
@@ -53,43 +54,43 @@ public class CrawlBizService { @@ -53,43 +54,43 @@ public class CrawlBizService {
53 /** 54 /**
54 * 登陆准备 55 * 登陆准备
55 * 56 *
56 - * @param phoneNo 账户号  
57 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 57 + * @param accountNo 账户号
  58 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
58 * @return 59 * @return
59 */ 60 */
60 - public String preLogin(String phoneNo, Integer type) { 61 + public String preLogin(String accountNo, Integer type) {
61 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); 62 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
62 AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); 63 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
63 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); 64 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
64 - return crawlStrategy.preLogin(phoneNo); 65 + return crawlStrategy.preLogin(accountNo);
65 } 66 }
66 67
67 /** 68 /**
68 * 登陆 69 * 登陆
69 * 70 *
70 - * @param phoneNo 账户号  
71 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 71 + * @param accountNo 账户号
  72 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
72 * @return 73 * @return
73 */ 74 */
74 - public boolean doLogin(String phoneNo, Integer type) { 75 + public boolean doLogin(String accountNo, Integer type) {
75 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); 76 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
76 AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); 77 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
77 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); 78 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
78 - return crawlStrategy.doLogin(phoneNo); 79 + return crawlStrategy.doLogin(accountNo);
79 } 80 }
80 81
81 /** 82 /**
82 * 退出浏览器 83 * 退出浏览器
83 * 84 *
84 - * @param phoneNo 账户号  
85 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 85 + * @param accountNo 账户号
  86 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
86 * @return 87 * @return
87 */ 88 */
88 - public boolean exitBrowser(String phoneNo, Integer type) { 89 + public boolean exitBrowser(String accountNo, Integer type) {
89 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); 90 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
90 AssertUtil.notNull(typeEnum, () -> "平台类型不正确"); 91 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
91 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); 92 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
92 - return crawlStrategy.exitBrowser(phoneNo); 93 + return crawlStrategy.exitBrowser(accountNo);
93 } 94 }
94 95
95 /** 96 /**
@@ -99,13 +100,13 @@ public class CrawlBizService { @@ -99,13 +100,13 @@ public class CrawlBizService {
99 */ 100 */
100 public void crawlData(Account account) throws IOException { 101 public void crawlData(Account account) throws IOException {
101 Integer type = account.getType(); 102 Integer type = account.getType();
102 - final String accountNo = account.getPhoneNo(); 103 + final String accountNo = account.getAccountNo();
103 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); 104 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
104 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); 105 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
105 // 抓取数据 106 // 抓取数据
106 log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据"); 107 log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据");
107 - final Integer fansCnt = crawlStrategy.updateAccountFans(accountNo);// 更新粉丝数  
108 - if (Objects.isNull(fansCnt)) { 108 + final ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数
  109 + if (Objects.isNull(accountMsg)) {
109 doPushExpireAccount(account); 110 doPushExpireAccount(account);
110 return; 111 return;
111 } 112 }
@@ -120,19 +121,20 @@ public class CrawlBizService { @@ -120,19 +121,20 @@ public class CrawlBizService {
120 return; 121 return;
121 } 122 }
122 // 上报数据 123 // 上报数据
123 - final boolean reportFansCnt = this.doReportFansCnt(account, fansCnt);  
124 - if (!reportFansCnt) {  
125 - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的粉丝数据失败"); 124 + final boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg);
  125 + final String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo);
  126 + if (!reportAccountMsg) {
  127 + log.error(format + "的账户信息失败");
126 return; 128 return;
127 } 129 }
128 final boolean reportVideo = this.doReportVideo(account, allVideoMsg); 130 final boolean reportVideo = this.doReportVideo(account, allVideoMsg);
129 if (!reportVideo) { 131 if (!reportVideo) {
130 - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的视频数据失败"); 132 + log.error(format + "的视频数据失败");
131 return; 133 return;
132 } 134 }
133 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); 135 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
134 if (!reportLive) { 136 if (!reportLive) {
135 - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); 137 + log.error(format + "的直播数据失败");
136 return; 138 return;
137 } 139 }
138 this.afterCrawl(accountService.findById(account.getId())); 140 this.afterCrawl(accountService.findById(account.getId()));
@@ -158,24 +160,24 @@ public class CrawlBizService { @@ -158,24 +160,24 @@ public class CrawlBizService {
158 int hasTryTimes = 0; 160 int hasTryTimes = 0;
159 int maxTryTimes = 2; 161 int maxTryTimes = 2;
160 while (!result && hasTryTimes < maxTryTimes) { 162 while (!result && hasTryTimes < maxTryTimes) {
161 - result = accountRpcService.pushExpireAccount(account.getPhoneNo(), account.getType()); 163 + result = accountRpcService.pushExpireAccount(account.getAccountNo(), account.getType());
162 hasTryTimes++; 164 hasTryTimes++;
163 } 165 }
164 } 166 }
165 167
166 /** 168 /**
167 - * 上报粉丝数 169 + * 上报账户信息
168 * 170 *
169 - * @param account 账户  
170 - * @param fansCnt 粉丝数 171 + * @param account 账户
  172 + * @param accountMsg 账户其他信息
171 * @return 173 * @return
172 */ 174 */
173 - private boolean doReportFansCnt(Account account, Integer fansCnt) { 175 + private boolean doReportAccountMsg(Account account, ReportAccountDto accountMsg) {
174 boolean reportFansCnt; 176 boolean reportFansCnt;
175 int hasTryTimes = 0; 177 int hasTryTimes = 0;
176 int maxTryTimes = 2; 178 int maxTryTimes = 2;
177 while (hasTryTimes < maxTryTimes) { 179 while (hasTryTimes < maxTryTimes) {
178 - reportFansCnt = reportRpcService.reportFansCnt(account.getPhoneNo(), account.getType(), fansCnt); 180 + reportFansCnt = reportRpcService.reportAccountMsg(account.getAccountNo(), account.getType(), accountMsg);
179 if (reportFansCnt) { 181 if (reportFansCnt) {
180 return true; 182 return true;
181 } else { 183 } else {
@@ -198,7 +200,7 @@ public class CrawlBizService { @@ -198,7 +200,7 @@ public class CrawlBizService {
198 int hasTryTimes = 0; 200 int hasTryTimes = 0;
199 int maxTryTimes = 2; 201 int maxTryTimes = 2;
200 while (hasTryTimes < maxTryTimes) { 202 while (hasTryTimes < maxTryTimes) {
201 - reportVideo = reportRpcService.reportVideo(account.getPhoneNo(), account.getType(), allVideoMsg); 203 + reportVideo = reportRpcService.reportVideo(account.getAccountNo(), account.getType(), allVideoMsg);
202 if (reportVideo) { 204 if (reportVideo) {
203 return true; 205 return true;
204 } else { 206 } else {
@@ -221,7 +223,7 @@ public class CrawlBizService { @@ -221,7 +223,7 @@ public class CrawlBizService {
221 int hasTryTimes = 0; 223 int hasTryTimes = 0;
222 int maxTryTimes = 2; 224 int maxTryTimes = 2;
223 while (hasTryTimes < maxTryTimes) { 225 while (hasTryTimes < maxTryTimes) {
224 - reportLive = reportRpcService.reportLive(account.getPhoneNo(), account.getType(), yesterdayLiveMsg); 226 + reportLive = reportRpcService.reportLive(account.getAccountNo(), account.getType(), yesterdayLiveMsg);
225 if (reportLive) { 227 if (reportLive) {
226 return true; 228 return true;
227 } else { 229 } else {
@@ -243,14 +245,14 @@ public class CrawlBizService { @@ -243,14 +245,14 @@ public class CrawlBizService {
243 public void reportLive(Account account) throws IOException { 245 public void reportLive(Account account) throws IOException {
244 log.info(Thread.currentThread().getName() + " spring监听器在CrawlBizService.reportLive()方法上成功收到消息: " + account); 246 log.info(Thread.currentThread().getName() + " spring监听器在CrawlBizService.reportLive()方法上成功收到消息: " + account);
245 Integer type = account.getType(); 247 Integer type = account.getType();
246 - final String accountNo = account.getPhoneNo(); 248 + final String accountNo = account.getAccountNo();
247 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type); 249 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
248 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum); 250 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
249 final List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息 251 final List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息
250 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg); 252 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
251 if (!reportLive) { 253 if (!reportLive) {
252 accountService.setAccountUndone(accountNo); 254 accountService.setAccountUndone(accountNo);
253 - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败"); 255 + log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + AccountTypeEnum.getNameByValue(account.getType()) + "的直播数据失败");
254 } 256 }
255 } 257 }
256 } 258 }
src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java
  View file @d472478
@@ -3,6 +3,7 @@ package cn.fw.freya.service.crawl; @@ -3,6 +3,7 @@ package cn.fw.freya.service.crawl;
3 import cn.fw.freya.enums.AccountTypeEnum; 3 import cn.fw.freya.enums.AccountTypeEnum;
4 import cn.fw.freya.model.data.pool.LivePool; 4 import cn.fw.freya.model.data.pool.LivePool;
5 import cn.fw.freya.model.data.pool.VideoPool; 5 import cn.fw.freya.model.data.pool.VideoPool;
  6 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
6 import cn.fw.freya.utils.http.InitUserAgent; 7 import cn.fw.freya.utils.http.InitUserAgent;
7 8
8 import java.io.IOException; 9 import java.io.IOException;
@@ -102,11 +103,11 @@ public interface CrawlStrategy { @@ -102,11 +103,11 @@ public interface CrawlStrategy {
102 List<LivePool> getYesterdayLiveMsg(String accountNo) throws IOException; 103 List<LivePool> getYesterdayLiveMsg(String accountNo) throws IOException;
103 104
104 /** 105 /**
105 - * 更新账户粉丝数 106 + * 更新账户信息
106 * 107 *
107 * @param accountNo 账户号 108 * @param accountNo 账户号
108 * @return 109 * @return
109 * @throws IOException 110 * @throws IOException
110 */ 111 */
111 - Integer updateAccountFans(String accountNo) throws IOException; 112 + ReportAccountDto updateAccountMsg(String accountNo) throws IOException;
112 } 113 }
src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java
  View file @d472478
@@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account; @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account;
10 import cn.fw.freya.model.data.FwCookie; 10 import cn.fw.freya.model.data.FwCookie;
11 import cn.fw.freya.model.data.pool.LivePool; 11 import cn.fw.freya.model.data.pool.LivePool;
12 import cn.fw.freya.model.data.pool.VideoPool; 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 import cn.fw.freya.service.crawl.CrawlStrategy; 14 import cn.fw.freya.service.crawl.CrawlStrategy;
14 import cn.fw.freya.service.data.AccountService; 15 import cn.fw.freya.service.data.AccountService;
15 import cn.fw.freya.utils.DateUtil; 16 import cn.fw.freya.utils.DateUtil;
@@ -154,7 +155,7 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -154,7 +155,7 @@ public class BilibiliCrawl implements CrawlStrategy {
154 } 155 }
155 } while (currentPage * pageSize < total);// 条件为false, 退出 156 } while (currentPage * pageSize < total);// 条件为false, 退出
156 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray))); 157 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray)));
157 - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue()); 158 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue());
158 // 视频数据存库 159 // 视频数据存库
159 List<VideoPool> videoPoolList = new ArrayList<>(); 160 List<VideoPool> videoPoolList = new ArrayList<>();
160 StringBuilder sb = new StringBuilder(); 161 StringBuilder sb = new StringBuilder();
@@ -189,7 +190,7 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -189,7 +190,7 @@ public class BilibiliCrawl implements CrawlStrategy {
189 .playCount(playCnt)// 播放次数 190 .playCount(playCnt)// 播放次数
190 .likeCount(Optional.ofNullable(stat.getInteger("like")).orElse(0))// 点赞数 191 .likeCount(Optional.ofNullable(stat.getInteger("like")).orElse(0))// 点赞数
191 .commentCount(Optional.ofNullable(stat.getInteger("reply")).orElse(0))// 评论数 192 .commentCount(Optional.ofNullable(stat.getInteger("reply")).orElse(0))// 评论数
192 - .phoneNo(accountNo) 193 + .accountNo(accountNo)
193 .reportDate(new Date()) 194 .reportDate(new Date())
194 .publishTime(new Date(Archive.getLong("ctime") * 1000))// 发布时间 195 .publishTime(new Date(Archive.getLong("ctime") * 1000))// 发布时间
195 .videoUrl("https://www.bilibili.com/video/" + videoId)// 视频播放地址 196 .videoUrl("https://www.bilibili.com/video/" + videoId)// 视频播放地址
@@ -207,7 +208,7 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -207,7 +208,7 @@ public class BilibiliCrawl implements CrawlStrategy {
207 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 208 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
208 } else { 209 } else {
209 final VideoPool nullVideo = VideoPool.builder() 210 final VideoPool nullVideo = VideoPool.builder()
210 - .phoneNo(accountNo) 211 + .accountNo(accountNo)
211 .type(this.getType().getValue()) 212 .type(this.getType().getValue())
212 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 213 .resourceType(ResourceTypeEnum.VIDEO.getValue())
213 .reportDate(new Date()) 214 .reportDate(new Date())
@@ -367,7 +368,7 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -367,7 +368,7 @@ public class BilibiliCrawl implements CrawlStrategy {
367 */ 368 */
368 livePoolList.add(LivePool.builder() 369 livePoolList.add(LivePool.builder()
369 .type(this.getType().getValue()) 370 .type(this.getType().getValue())
370 - .phoneNo(accountNo) 371 + .accountNo(accountNo)
371 .reportDate(new Date()) 372 .reportDate(new Date())
372 .commentUserCnt(Optional.ofNullable(obj.getInteger("danmu_num")).orElse(0))// 评论数 373 .commentUserCnt(Optional.ofNullable(obj.getInteger("danmu_num")).orElse(0))// 评论数
373 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) 374 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
@@ -400,7 +401,7 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -400,7 +401,7 @@ public class BilibiliCrawl implements CrawlStrategy {
400 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的Bilibili直播数据"); 401 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的Bilibili直播数据");
401 final LivePool nullLive = LivePool.builder() 402 final LivePool nullLive = LivePool.builder()
402 .type(this.getType().getValue()) 403 .type(this.getType().getValue())
403 - .phoneNo(accountNo) 404 + .accountNo(accountNo)
404 .reportDate(new Date()) 405 .reportDate(new Date())
405 .build(); 406 .build();
406 livePoolDao.save(nullLive); 407 livePoolDao.save(nullLive);
@@ -419,10 +420,10 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -419,10 +420,10 @@ public class BilibiliCrawl implements CrawlStrategy {
419 */ 420 */
420 @Override 421 @Override
421 @Transactional 422 @Transactional
422 - public Integer updateAccountFans(String accountNo) throws IOException {  
423 - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));  
424 - if (Objects.nonNull(hasFoundFansCnt)) {  
425 - return hasFoundFansCnt; 423 + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException {
  424 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  425 + if (Objects.nonNull(hasFoundAccountMsg)) {
  426 + return hasFoundAccountMsg;
426 } 427 }
427 HttpConfig config = HttpConfig.custom() 428 HttpConfig config = HttpConfig.custom()
428 .url("https://member.bilibili.com/x/web/index/stat") 429 .url("https://member.bilibili.com/x/web/index/stat")
@@ -442,28 +443,37 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -442,28 +443,37 @@ public class BilibiliCrawl implements CrawlStrategy {
442 if (!StringUtils.hasText(res)) { 443 if (!StringUtils.hasText(res)) {
443 throw new BusinessException("调用Bilibili[粉丝]接口失败"); 444 throw new BusinessException("调用Bilibili[粉丝]接口失败");
444 } 445 }
445 - assert response != null;  
446 - Integer fansCnt; 446 + ReportAccountDto reportAccountDto;
447 try { 447 try {
448 - fansCnt = response.getJSONObject("data").getInteger("total_fans"); 448 + JSONObject data = response.getJSONObject("data");
  449 + reportAccountDto = ReportAccountDto.builder()
  450 + .fansCnt(data.getInteger("total_fans"))
  451 + //.followCnt(data.getInteger("followCnt"))
  452 + .likeCnt(data.getInteger("total_like"))
  453 + //.accountId(data.getString("userId"))
  454 + //.userOtherId(data.getString("userKwaiId"))
  455 + //.accountName(data.getString("userName"))
  456 + .build();
449 } catch (Exception e) { 457 } catch (Exception e) {
450 log.error("获取[哔哩哔哩]fansCnt发生错误", e); 458 log.error("获取[哔哩哔哩]fansCnt发生错误", e);
451 - fansCnt = 0; 459 + reportAccountDto = ReportAccountDto.builder()
  460 + .fansCnt(0)
  461 + .build();
452 } 462 }
453 - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());  
454 - if (Objects.nonNull(account)) {  
455 - accountDao.updateFans(account.getId(), fansCnt, new Date());  
456 - } else { 463 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  464 + if (Objects.nonNull(account))
  465 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
  466 + else
457 accountDao.save(Account.builder() 467 accountDao.save(Account.builder()
458 .cookiesStatus(true) 468 .cookiesStatus(true)
459 - .phoneNo(accountNo) 469 + .accountNo(accountNo)
460 .type(this.getType().getValue()) 470 .type(this.getType().getValue())
461 - .fansCnt(fansCnt) 471 + .fansCnt(reportAccountDto.getFansCnt())
  472 + .accountName(reportAccountDto.getAccountName())
462 .reportDate(new Date()) 473 .reportDate(new Date())
463 .done(false) 474 .done(false)
464 .build()); 475 .build());
465 - }  
466 - return fansCnt; 476 + return reportAccountDto;
467 } 477 }
468 478
469 /** 479 /**
@@ -563,11 +573,11 @@ public class BilibiliCrawl implements CrawlStrategy { @@ -563,11 +573,11 @@ public class BilibiliCrawl implements CrawlStrategy {
563 /** 573 /**
564 * 根据账户号查到用户cookies, 并组装成String形式返回 574 * 根据账户号查到用户cookies, 并组装成String形式返回
565 * 575 *
566 - * @param phoneNo 账户号 576 + * @param accountNo 账户号
567 * @return 577 * @return
568 */ 578 */
569 - public String getUserCookies(String phoneNo) {  
570 - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue())); 579 + public String getUserCookies(String accountNo) {
  580 + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue()));
571 } 581 }
572 582
573 /** 583 /**
src/main/java/cn/fw/freya/service/crawl/impl/Common.java
  View file @d472478
@@ -10,6 +10,7 @@ import cn.fw.freya.model.data.FwCookie; @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.FwCookie;
10 import cn.fw.freya.model.data.ResponseReceived; 10 import cn.fw.freya.model.data.ResponseReceived;
11 import cn.fw.freya.model.data.pool.LivePool; 11 import cn.fw.freya.model.data.pool.LivePool;
12 import cn.fw.freya.model.data.pool.VideoPool; 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 import com.alibaba.fastjson.JSON; 14 import com.alibaba.fastjson.JSON;
14 import com.alibaba.fastjson.JSONObject; 15 import com.alibaba.fastjson.JSONObject;
15 import lombok.RequiredArgsConstructor; 16 import lombok.RequiredArgsConstructor;
@@ -159,7 +160,7 @@ public class Common { @@ -159,7 +160,7 @@ public class Common {
159 * @param type 账户类型 160 * @param type 账户类型
160 */ 161 */
161 public void deleteCookies(String accountNo, Integer type) { 162 public void deleteCookies(String accountNo, Integer type) {
162 - cookieDao.deleteByPhoneNoAndType(accountNo, type); 163 + cookieDao.deleteByAccountNoAndType(accountNo, type);
163 } 164 }
164 165
165 /** 166 /**
@@ -169,7 +170,7 @@ public class Common { @@ -169,7 +170,7 @@ public class Common {
169 * @return 170 * @return
170 */ 171 */
171 public List<FwCookie> loadCookie(String accountNo, Integer type) { 172 public List<FwCookie> loadCookie(String accountNo, Integer type) {
172 - return Optional.ofNullable(cookieDao.findByPhoneNoAndType(accountNo, type)) 173 + return Optional.ofNullable(cookieDao.findByAccountNoAndType(accountNo, type))
173 .orElse(new ArrayList<>()); 174 .orElse(new ArrayList<>());
174 } 175 }
175 176
@@ -252,10 +253,14 @@ public class Common { @@ -252,10 +253,14 @@ public class Common {
252 * @param reportDate 指定日期 253 * @param reportDate 指定日期
253 * @return 254 * @return
254 */ 255 */
255 - public Integer getHasFoundFansCnt(String accountNo, Integer type, Date reportDate) {  
256 - final Integer hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate);  
257 - if (Objects.nonNull(hasReportDate)) {  
258 - return hasReportDate; 256 + public ReportAccountDto getHasFoundAccountMsg(String accountNo, Integer type, Date reportDate) {
  257 + final List<Account> hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate);
  258 + if (!CollectionUtils.isEmpty(hasReportDate)) {
  259 + final Account account = hasReportDate.get(0);
  260 + return ReportAccountDto.builder()
  261 + .fansCnt(account.getFansCnt())
  262 + .accountName(account.getAccountName())
  263 + .build();
259 } 264 }
260 return null; 265 return null;
261 } 266 }
@@ -269,9 +274,9 @@ public class Common { @@ -269,9 +274,9 @@ public class Common {
269 * @return 274 * @return
270 */ 275 */
271 public List<VideoPool> getHasFoundVideo(String accountNo, Integer type, Date reportDate) { 276 public List<VideoPool> getHasFoundVideo(String accountNo, Integer type, Date reportDate) {
272 - final List<VideoPool> hasReportDate = videoPoolDao.getHasReportDate(accountNo, type, reportDate);  
273 - if (hasReportDate.size() > 0) {  
274 - return hasReportDate; 277 + final List<VideoPool> hasReportData = videoPoolDao.getHasReportDate(accountNo, type, reportDate);
  278 + if (hasReportData.size() > 0) {
  279 + return hasReportData;
275 } 280 }
276 return null; 281 return null;
277 } 282 }
@@ -285,9 +290,9 @@ public class Common { @@ -285,9 +290,9 @@ public class Common {
285 * @return 290 * @return
286 */ 291 */
287 public List<LivePool> getHasFoundLive(String accountNo, Integer type, Date reportDate) { 292 public List<LivePool> getHasFoundLive(String accountNo, Integer type, Date reportDate) {
288 - final List<LivePool> hasReportDate = livePoolDao.getHasReportDate(accountNo, type, reportDate);  
289 - if (hasReportDate.size() > 0) {  
290 - return hasReportDate; 293 + final List<LivePool> hasReportData = livePoolDao.getHasReportData(accountNo, type, reportDate);
  294 + if (hasReportData.size() > 0) {
  295 + return hasReportData;
291 } 296 }
292 return null; 297 return null;
293 } 298 }
src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java
  View file @d472478
@@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account; @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account;
10 import cn.fw.freya.model.data.FwCookie; 10 import cn.fw.freya.model.data.FwCookie;
11 import cn.fw.freya.model.data.pool.LivePool; 11 import cn.fw.freya.model.data.pool.LivePool;
12 import cn.fw.freya.model.data.pool.VideoPool; 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 import cn.fw.freya.service.crawl.CrawlStrategy; 14 import cn.fw.freya.service.crawl.CrawlStrategy;
14 import cn.fw.freya.service.data.AccountService; 15 import cn.fw.freya.service.data.AccountService;
15 import cn.fw.freya.utils.DateUtil; 16 import cn.fw.freya.utils.DateUtil;
@@ -183,7 +184,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -183,7 +184,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
183 assert response1 != null; 184 assert response1 != null;
184 JSONArray smallVideoJsonArray = Optional.ofNullable(response1.getJSONArray("item_datas")).orElse(new JSONArray());// 获取[小视频]数据数组 185 JSONArray smallVideoJsonArray = Optional.ofNullable(response1.getJSONArray("item_datas")).orElse(new JSONArray());// 获取[小视频]数据数组
185 videoJsonArray.addAll(smallVideoJsonArray); 186 videoJsonArray.addAll(smallVideoJsonArray);
186 - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue()); 187 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue());
187 // 视频数据存库 188 // 视频数据存库
188 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size()); 189 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size());
189 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray))); 190 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray)));
@@ -215,7 +216,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -215,7 +216,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
215 .playCount(playCount)// 播放次数 216 .playCount(playCount)// 播放次数
216 .likeCount(Optional.ofNullable(DCSObj.getInteger("digg_count")).orElse(0))// 点赞数 217 .likeCount(Optional.ofNullable(DCSObj.getInteger("digg_count")).orElse(0))// 点赞数
217 .commentCount(Optional.ofNullable(DCSObj.getInteger("comment_count")).orElse(0))// 评论数 218 .commentCount(Optional.ofNullable(DCSObj.getInteger("comment_count")).orElse(0))// 评论数
218 - .phoneNo(accountNo) 219 + .accountNo(accountNo)
219 .reportDate(new Date()) 220 .reportDate(new Date())
220 .publishTime(new Date(obj.getLong("create_time") * 1000L))// 发布时间 221 .publishTime(new Date(obj.getLong("create_time") * 1000L))// 发布时间
221 .videoUrl("https://www.ixigua.com/" + videoId)// 播放地址 222 .videoUrl("https://www.ixigua.com/" + videoId)// 播放地址
@@ -232,7 +233,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -232,7 +233,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
232 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 233 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
233 } else { 234 } else {
234 final VideoPool nullVideo = VideoPool.builder() 235 final VideoPool nullVideo = VideoPool.builder()
235 - .phoneNo(accountNo) 236 + .accountNo(accountNo)
236 .type(this.getType().getValue()) 237 .type(this.getType().getValue())
237 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 238 .resourceType(ResourceTypeEnum.VIDEO.getValue())
238 .reportDate(new Date()) 239 .reportDate(new Date())
@@ -285,7 +286,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -285,7 +286,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
285 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据"); 286 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据");
286 final LivePool nullLive = LivePool.builder() 287 final LivePool nullLive = LivePool.builder()
287 .type(this.getType().getValue()) 288 .type(this.getType().getValue())
288 - .phoneNo(accountNo) 289 + .accountNo(accountNo)
289 .reportDate(new Date()) 290 .reportDate(new Date())
290 .build(); 291 .build();
291 livePoolDao.save(nullLive); 292 livePoolDao.save(nullLive);
@@ -326,7 +327,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -326,7 +327,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
326 assert userInfo != null; 327 assert userInfo != null;
327 livePoolList.add(LivePool.builder() 328 livePoolList.add(LivePool.builder()
328 .type(this.getType().getValue()) 329 .type(this.getType().getValue())
329 - .phoneNo(accountNo) 330 + .accountNo(accountNo)
330 .reportDate(new Date()) 331 .reportDate(new Date())
331 .commentUserCnt(commentCnt)// 评论数 332 .commentUserCnt(commentCnt)// 评论数
332 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) 333 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
@@ -359,7 +360,7 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -359,7 +360,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
359 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据"); 360 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据");
360 final LivePool nullLive = LivePool.builder() 361 final LivePool nullLive = LivePool.builder()
361 .type(this.getType().getValue()) 362 .type(this.getType().getValue())
362 - .phoneNo(accountNo) 363 + .accountNo(accountNo)
363 .reportDate(new Date()) 364 .reportDate(new Date())
364 .build(); 365 .build();
365 livePoolDao.save(nullLive); 366 livePoolDao.save(nullLive);
@@ -378,14 +379,19 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -378,14 +379,19 @@ public class DongCheDiCrawl implements CrawlStrategy {
378 * @return 379 * @return
379 */ 380 */
380 @Override 381 @Override
381 - public Integer updateAccountFans(String accountNo) {  
382 - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));  
383 - if (Objects.nonNull(hasFoundFansCnt)) {  
384 - return hasFoundFansCnt; 382 + public ReportAccountDto updateAccountMsg(String accountNo) {
  383 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  384 + if (Objects.nonNull(hasFoundAccountMsg)) {
  385 + return hasFoundAccountMsg;
385 } 386 }
386 final String userId; 387 final String userId;
  388 + String userNick;
  389 + String ttId;
387 try { 390 try {
388 - userId = Objects.requireNonNull(this.getUserInfo(accountNo)).getString("userId"); 391 + final JSONObject obj = Objects.requireNonNull(this.getUserInfo(accountNo));
  392 + userId = obj.getString("userId");
  393 + userNick = obj.getString("userNick");
  394 + ttId = obj.getString("ttId");
389 } catch (NullPointerException e) { 395 } catch (NullPointerException e) {
390 return null; 396 return null;
391 } 397 }
@@ -414,24 +420,32 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -414,24 +420,32 @@ public class DongCheDiCrawl implements CrawlStrategy {
414 throw new BusinessException("调用懂车帝[粉丝]接口失败"); 420 throw new BusinessException("调用懂车帝[粉丝]接口失败");
415 } 421 }
416 final JSONArray data = response1.getJSONArray("data"); 422 final JSONArray data = response1.getJSONArray("data");
  423 + ReportAccountDto reportAccountDto;
417 for (Object item : data) { 424 for (Object item : data) {
418 JSONObject obj = (JSONObject) item; 425 JSONObject obj = (JSONObject) item;
419 if (obj.getString("title").contains("总数")) { 426 if (obj.getString("title").contains("总数")) {
420 - final Integer fansCnt = obj.getInteger("data");  
421 - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());  
422 - if (Objects.nonNull(account)) {  
423 - accountDao.updateFans(account.getId(), fansCnt, new Date());  
424 - } else { 427 + reportAccountDto = ReportAccountDto.builder()
  428 + .fansCnt(obj.getInteger("data"))
  429 + //.followCnt(obj.getInteger("followCnt"))
  430 + //.likeCnt(obj.getInteger("likeCnt"))
  431 + .accountId(userId)
  432 + .userOtherId(ttId)
  433 + .accountName(userNick)
  434 + .build();
  435 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  436 + if (Objects.nonNull(account))
  437 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
  438 + else
425 accountDao.save(Account.builder() 439 accountDao.save(Account.builder()
426 .cookiesStatus(true) 440 .cookiesStatus(true)
427 - .phoneNo(accountNo) 441 + .accountNo(accountNo)
428 .type(this.getType().getValue()) 442 .type(this.getType().getValue())
429 - .fansCnt(fansCnt) 443 + .fansCnt(reportAccountDto.getFansCnt())
  444 + .accountName(reportAccountDto.getAccountName())
430 .reportDate(new Date()) 445 .reportDate(new Date())
431 .done(false) 446 .done(false)
432 .build()); 447 .build());
433 - }  
434 - return fansCnt; 448 + return reportAccountDto;
435 } 449 }
436 } 450 }
437 return null; 451 return null;
@@ -559,11 +573,11 @@ public class DongCheDiCrawl implements CrawlStrategy { @@ -559,11 +573,11 @@ public class DongCheDiCrawl implements CrawlStrategy {
559 /** 573 /**
560 * 根据账户号查到用户cookies, 并组装成String形式返回 574 * 根据账户号查到用户cookies, 并组装成String形式返回
561 * 575 *
562 - * @param phoneNo 账户号 576 + * @param accountNo 账户号
563 * @return 577 * @return
564 */ 578 */
565 - public String getUserCookies(String phoneNo) {  
566 - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue())); 579 + public String getUserCookies(String accountNo) {
  580 + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue()));
567 } 581 }
568 582
569 /** 583 /**
src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java
  View file @d472478
@@ -13,6 +13,7 @@ import cn.fw.freya.model.data.LiveOverview; @@ -13,6 +13,7 @@ import cn.fw.freya.model.data.LiveOverview;
13 import cn.fw.freya.model.data.ResponseReceived; 13 import cn.fw.freya.model.data.ResponseReceived;
14 import cn.fw.freya.model.data.pool.LivePool; 14 import cn.fw.freya.model.data.pool.LivePool;
15 import cn.fw.freya.model.data.pool.VideoPool; 15 import cn.fw.freya.model.data.pool.VideoPool;
  16 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
16 import cn.fw.freya.service.crawl.CrawlStrategy; 17 import cn.fw.freya.service.crawl.CrawlStrategy;
17 import cn.fw.freya.service.data.AccountService; 18 import cn.fw.freya.service.data.AccountService;
18 import cn.fw.freya.utils.DateUtil; 19 import cn.fw.freya.utils.DateUtil;
@@ -132,13 +133,13 @@ public class DouYinCrawl implements CrawlStrategy { @@ -132,13 +133,13 @@ public class DouYinCrawl implements CrawlStrategy {
132 */ 133 */
133 @Override 134 @Override
134 @Transactional 135 @Transactional
135 - /*public List<VideoPool> getAllVideoMsg(String phoneNo) {  
136 - final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(phoneNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date())); 136 + /*public List<VideoPool> getAllVideoMsg(String accountNo) {
  137 + final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
137 if (Objects.nonNull(hasFoundVideo)) { 138 if (Objects.nonNull(hasFoundVideo)) {
138 return hasFoundVideo; 139 return hasFoundVideo;
139 } 140 }
140 final String uuid = UUID.randomUUID().toString().replace("-", ""); 141 final String uuid = UUID.randomUUID().toString().replace("-", "");
141 - final WebDriver driver = this.getDYDriver(phoneNo, uuid); 142 + final WebDriver driver = this.getDYDriver(accountNo, uuid);
142 String targetUrl = "https://creator.douyin.com/creator-micro/content/manage"; 143 String targetUrl = "https://creator.douyin.com/creator-micro/content/manage";
143 driver.get(targetUrl); 144 driver.get(targetUrl);
144 List<WebElement> endNodes; 145 List<WebElement> endNodes;
@@ -146,7 +147,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -146,7 +147,7 @@ public class DouYinCrawl implements CrawlStrategy {
146 do { 147 do {
147 endNodes = driver.findElements(By.xpath("//div[starts-with(@class,'load-more')]/div[text()='没有更多视频']")); 148 endNodes = driver.findElements(By.xpath("//div[starts-with(@class,'load-more')]/div[text()='没有更多视频']"));
148 if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) { 149 if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
149 - this.exitBrowser(phoneNo, uuid); 150 + this.exitBrowser(accountNo, uuid);
150 return null; 151 return null;
151 } 152 }
152 if (CollectionUtils.isEmpty(endNodes)) { 153 if (CollectionUtils.isEmpty(endNodes)) {
@@ -211,7 +212,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -211,7 +212,7 @@ public class DouYinCrawl implements CrawlStrategy {
211 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString()) 212 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
212 .duration(jsonObj.getLong("duration") / (1000 * 1.0)) 213 .duration(jsonObj.getLong("duration") / (1000 * 1.0))
213 .likeCount(statistics.getInteger("share_count")) 214 .likeCount(statistics.getInteger("share_count"))
214 - .phoneNo(phoneNo) 215 + .accountNo(accountNo)
215 .playCount(statistics.getInteger("play_count")) 216 .playCount(statistics.getInteger("play_count"))
216 .reportDate(new Date()) 217 .reportDate(new Date())
217 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1) 218 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
@@ -234,7 +235,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -234,7 +235,7 @@ public class DouYinCrawl implements CrawlStrategy {
234 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 235 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
235 } else { 236 } else {
236 videoPoolDao.save(VideoPool.builder() 237 videoPoolDao.save(VideoPool.builder()
237 - .phoneNo(phoneNo) 238 + .accountNo(accountNo)
238 .type(this.getType().getValue()) 239 .type(this.getType().getValue())
239 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 240 .resourceType(ResourceTypeEnum.VIDEO.getValue())
240 .reportDate(new Date()) 241 .reportDate(new Date())
@@ -243,9 +244,9 @@ public class DouYinCrawl implements CrawlStrategy { @@ -243,9 +244,9 @@ public class DouYinCrawl implements CrawlStrategy {
243 log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息"); 244 log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息");
244 } catch (Exception e) { 245 } catch (Exception e) {
245 assert log != null; 246 assert log != null;
246 - log.error(LocalDate.now() + " 执行保存" + phoneNo + "的抖音视频数据失败", e); 247 + log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败", e);
247 } 248 }
248 - this.exitBrowser(phoneNo, uuid); 249 + this.exitBrowser(accountNo, uuid);
249 return videoPoolList; 250 return videoPoolList;
250 }*/ 251 }*/
251 public List<VideoPool> getAllVideoMsg(String accountNo) { 252 public List<VideoPool> getAllVideoMsg(String accountNo) {
@@ -279,7 +280,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -279,7 +280,7 @@ public class DouYinCrawl implements CrawlStrategy {
279 if (element.getText().contains("没有") || element.getText().contains("起开始展示数据")) { 280 if (element.getText().contains("没有") || element.getText().contains("起开始展示数据")) {
280 this.exitBrowser(accountNo, uuid); 281 this.exitBrowser(accountNo, uuid);
281 videoPoolDao.save(VideoPool.builder() 282 videoPoolDao.save(VideoPool.builder()
282 - .phoneNo(accountNo) 283 + .accountNo(accountNo)
283 .type(this.getType().getValue()) 284 .type(this.getType().getValue())
284 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 285 .resourceType(ResourceTypeEnum.VIDEO.getValue())
285 .reportDate(new Date()) 286 .reportDate(new Date())
@@ -358,7 +359,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -358,7 +359,7 @@ public class DouYinCrawl implements CrawlStrategy {
358 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString()) 359 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
359 .duration(Optional.ofNullable(jsonObj.getDouble("duration")).orElse(0d) / 1000) 360 .duration(Optional.ofNullable(jsonObj.getDouble("duration")).orElse(0d) / 1000)
360 .likeCount(Optional.ofNullable(statistics.getInteger("digg_count")).orElse(0)) 361 .likeCount(Optional.ofNullable(statistics.getInteger("digg_count")).orElse(0))
361 - .phoneNo(accountNo) 362 + .accountNo(accountNo)
362 .playCount(Optional.ofNullable(statistics.getInteger("play_count")).orElse(0)) 363 .playCount(Optional.ofNullable(statistics.getInteger("play_count")).orElse(0))
363 .reportDate(new Date()) 364 .reportDate(new Date())
364 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1) 365 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
@@ -390,7 +391,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -390,7 +391,7 @@ public class DouYinCrawl implements CrawlStrategy {
390 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 391 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
391 } else { 392 } else {
392 videoPoolDao.save(VideoPool.builder() 393 videoPoolDao.save(VideoPool.builder()
393 - .phoneNo(accountNo) 394 + .accountNo(accountNo)
394 .type(this.getType().getValue()) 395 .type(this.getType().getValue())
395 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 396 .resourceType(ResourceTypeEnum.VIDEO.getValue())
396 .reportDate(new Date()) 397 .reportDate(new Date())
@@ -449,7 +450,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -449,7 +450,7 @@ public class DouYinCrawl implements CrawlStrategy {
449 .type(this.getType().getValue()) 450 .type(this.getType().getValue())
450 .reportDate(new Date()) 451 .reportDate(new Date())
451 .uploadTime(new Date()) 452 .uploadTime(new Date())
452 - .phoneNo(accountNo) 453 + .accountNo(accountNo)
453 .liveCnt(summarizeData.getInteger("yes_live_cnt")) 454 .liveCnt(summarizeData.getInteger("yes_live_cnt"))
454 .liveDuration(summarizeData.getDouble("yes_live_duration")) 455 .liveDuration(summarizeData.getDouble("yes_live_duration"))
455 .watchTimesCnt(summarizeData.getInteger("yes_watch_cnt")) 456 .watchTimesCnt(summarizeData.getInteger("yes_watch_cnt"))
@@ -514,7 +515,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -514,7 +515,7 @@ public class DouYinCrawl implements CrawlStrategy {
514 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的抖音直播数据"); 515 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的抖音直播数据");
515 final LivePool nullLive = LivePool.builder() 516 final LivePool nullLive = LivePool.builder()
516 .type(this.getType().getValue()) 517 .type(this.getType().getValue())
517 - .phoneNo(accountNo) 518 + .accountNo(accountNo)
518 .reportDate(new Date()) 519 .reportDate(new Date())
519 .build(); 520 .build();
520 livePoolDao.save(nullLive); 521 livePoolDao.save(nullLive);
@@ -567,7 +568,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -567,7 +568,7 @@ public class DouYinCrawl implements CrawlStrategy {
567 */ 568 */
568 livePoolList.add(LivePool.builder() 569 livePoolList.add(LivePool.builder()
569 .type(this.getType().getValue()) 570 .type(this.getType().getValue())
570 - .phoneNo(accountNo) 571 + .accountNo(accountNo)
571 .reportDate(new Date()) 572 .reportDate(new Date())
572 .commentUserCnt(Optional.ofNullable(obj.getInteger("comment_ucnt")).orElse(0))// 评论人数 573 .commentUserCnt(Optional.ofNullable(obj.getInteger("comment_ucnt")).orElse(0))// 评论人数
573 .consumeUserCnt(Optional.ofNullable(obj.getInteger("consume_ucnt")).orElse(0))// 付费人数 574 .consumeUserCnt(Optional.ofNullable(obj.getInteger("consume_ucnt")).orElse(0))// 付费人数
@@ -613,7 +614,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -613,7 +614,7 @@ public class DouYinCrawl implements CrawlStrategy {
613 } else { 614 } else {
614 livePoolDao.save(LivePool.builder() 615 livePoolDao.save(LivePool.builder()
615 .type(this.getType().getValue()) 616 .type(this.getType().getValue())
616 - .phoneNo(accountNo) 617 + .accountNo(accountNo)
617 .reportDate(new Date()) 618 .reportDate(new Date())
618 .build()); 619 .build());
619 } 620 }
@@ -633,10 +634,10 @@ public class DouYinCrawl implements CrawlStrategy { @@ -633,10 +634,10 @@ public class DouYinCrawl implements CrawlStrategy {
633 */ 634 */
634 @Override 635 @Override
635 @Transactional 636 @Transactional
636 - public Integer updateAccountFans(String accountNo) {  
637 - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));  
638 - if (Objects.nonNull(hasFoundFansCnt)) {  
639 - return hasFoundFansCnt; 637 + public ReportAccountDto updateAccountMsg(String accountNo) {
  638 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  639 + if (Objects.nonNull(hasFoundAccountMsg)) {
  640 + return hasFoundAccountMsg;
640 } 641 }
641 final String uuid = UUID.randomUUID().toString().replace("-", ""); 642 final String uuid = UUID.randomUUID().toString().replace("-", "");
642 final WebDriver driver = this.getDYDriver(accountNo, uuid); 643 final WebDriver driver = this.getDYDriver(accountNo, uuid);
@@ -678,9 +679,9 @@ public class DouYinCrawl implements CrawlStrategy { @@ -678,9 +679,9 @@ public class DouYinCrawl implements CrawlStrategy {
678 // 读http日志, 处理数据 679 // 读http日志, 处理数据
679 final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver); 680 final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
680 String dataUrl = "https://creator.douyin.com/web/api/media/user/info";// 数据接口地址 681 String dataUrl = "https://creator.douyin.com/web/api/media/user/info";// 数据接口地址
681 - final Integer[] fansCnt = new Integer[1]; 682 + AtomicReference<ReportAccountDto> reportAccountDto = new AtomicReference<>();
682 responseReceivedEvents.forEach(item -> { 683 responseReceivedEvents.forEach(item -> {
683 - JSONObject userInfo; 684 + JSONObject data;
684 try { 685 try {
685 final HttpResponse response = common.getHttpResponse(driver, item, dataUrl); 686 final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
686 if (Objects.nonNull(response)) { 687 if (Objects.nonNull(response)) {
@@ -690,26 +691,32 @@ public class DouYinCrawl implements CrawlStrategy { @@ -690,26 +691,32 @@ public class DouYinCrawl implements CrawlStrategy {
690 if (Objects.isNull(body)) { 691 if (Objects.isNull(body)) {
691 return; 692 return;
692 } 693 }
693 - userInfo = body.getJSONObject("user");  
694 - if (Objects.isNull(userInfo)) { 694 + data = body.getJSONObject("user");
  695 + if (Objects.isNull(data)) {
695 return; 696 return;
696 } 697 }
697 - fansCnt[0] = userInfo.getInteger("follower_count");  
698 - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());  
699 - if (Objects.nonNull(fansCnt[0])) {  
700 - if (Objects.nonNull(account)) {  
701 - accountDao.updateFans(account.getId(), fansCnt[0], new Date());  
702 - } else {  
703 - accountDao.save(Account.builder()  
704 - .cookiesStatus(true)  
705 - .phoneNo(accountNo)  
706 - .type(this.getType().getValue())  
707 - .fansCnt(fansCnt[0])  
708 - .reportDate(new Date())  
709 - .done(false)  
710 - .build());  
711 - }  
712 - } 698 + reportAccountDto.set(ReportAccountDto.builder()
  699 + .fansCnt(data.getInteger("follower_count"))
  700 + .followCnt(data.getInteger("following_count"))
  701 + .likeCnt(data.getInteger("total_favorited"))
  702 + .accountId(data.getString("unique_id"))
  703 + .userOtherId(data.getString("uid"))
  704 + .accountName(data.getString("nickname"))
  705 + .build()
  706 + );
  707 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  708 + if (Objects.nonNull(account))
  709 + accountDao.updateMsg(account.getId(), reportAccountDto.get().getAccountName(), reportAccountDto.get().getFansCnt(), new Date());
  710 + else
  711 + accountDao.save(Account.builder()
  712 + .cookiesStatus(true)
  713 + .accountNo(accountNo)
  714 + .type(this.getType().getValue())
  715 + .fansCnt(reportAccountDto.get().getFansCnt())
  716 + .accountName(reportAccountDto.get().getAccountName())
  717 + .reportDate(new Date())
  718 + .done(false)
  719 + .build());
713 } 720 }
714 } catch (Exception e) { 721 } catch (Exception e) {
715 this.exitBrowser(accountNo, uuid); 722 this.exitBrowser(accountNo, uuid);
@@ -718,7 +725,7 @@ public class DouYinCrawl implements CrawlStrategy { @@ -718,7 +725,7 @@ public class DouYinCrawl implements CrawlStrategy {
718 } 725 }
719 }); 726 });
720 this.exitBrowser(accountNo, uuid); 727 this.exitBrowser(accountNo, uuid);
721 - return fansCnt[0]; 728 + return reportAccountDto.get();
722 } 729 }
723 730
724 /** 731 /**
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @d472478
@@ -12,6 +12,7 @@ import cn.fw.freya.model.data.FwCookie; @@ -12,6 +12,7 @@ import cn.fw.freya.model.data.FwCookie;
12 import cn.fw.freya.model.data.ResponseReceived; 12 import cn.fw.freya.model.data.ResponseReceived;
13 import cn.fw.freya.model.data.pool.LivePool; 13 import cn.fw.freya.model.data.pool.LivePool;
14 import cn.fw.freya.model.data.pool.VideoPool; 14 import cn.fw.freya.model.data.pool.VideoPool;
  15 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
15 import cn.fw.freya.service.crawl.CrawlStrategy; 16 import cn.fw.freya.service.crawl.CrawlStrategy;
16 import cn.fw.freya.service.data.AccountService; 17 import cn.fw.freya.service.data.AccountService;
17 import cn.fw.freya.utils.DateUtil; 18 import cn.fw.freya.utils.DateUtil;
@@ -30,20 +31,29 @@ import lombok.RequiredArgsConstructor; @@ -30,20 +31,29 @@ import lombok.RequiredArgsConstructor;
30 import lombok.extern.slf4j.Slf4j; 31 import lombok.extern.slf4j.Slf4j;
31 import org.apache.http.client.CookieStore; 32 import org.apache.http.client.CookieStore;
32 import org.apache.http.impl.client.BasicCookieStore; 33 import org.apache.http.impl.client.BasicCookieStore;
  34 +import org.apache.logging.log4j.util.PropertiesUtil;
33 import org.openqa.selenium.By; 35 import org.openqa.selenium.By;
34 import org.openqa.selenium.Cookie; 36 import org.openqa.selenium.Cookie;
35 import org.openqa.selenium.WebDriver; 37 import org.openqa.selenium.WebDriver;
36 import org.openqa.selenium.WebElement; 38 import org.openqa.selenium.WebElement;
37 import org.openqa.selenium.support.ui.WebDriverWait; 39 import org.openqa.selenium.support.ui.WebDriverWait;
  40 +import org.springframework.context.SmartLifecycle;
  41 +import org.springframework.core.io.ClassPathResource;
  42 +import org.springframework.core.io.support.PropertiesLoaderUtils;
38 import org.springframework.stereotype.Service; 43 import org.springframework.stereotype.Service;
39 import org.springframework.transaction.annotation.Transactional; 44 import org.springframework.transaction.annotation.Transactional;
40 import org.springframework.util.CollectionUtils; 45 import org.springframework.util.CollectionUtils;
41 import org.springframework.util.StringUtils; 46 import org.springframework.util.StringUtils;
42 47
43 import javax.annotation.Resource; 48 import javax.annotation.Resource;
  49 +import java.io.BufferedWriter;
  50 +import java.io.FileOutputStream;
44 import java.io.IOException; 51 import java.io.IOException;
  52 +import java.io.OutputStreamWriter;
45 import java.math.BigDecimal; 53 import java.math.BigDecimal;
46 import java.math.RoundingMode; 54 import java.math.RoundingMode;
  55 +import java.net.URLDecoder;
  56 +import java.nio.charset.StandardCharsets;
47 import java.time.LocalDate; 57 import java.time.LocalDate;
48 import java.time.LocalDateTime; 58 import java.time.LocalDateTime;
49 import java.util.*; 59 import java.util.*;
@@ -62,8 +72,9 @@ import java.util.stream.Collectors; @@ -62,8 +72,9 @@ import java.util.stream.Collectors;
62 @Service 72 @Service
63 @RequiredArgsConstructor 73 @RequiredArgsConstructor
64 @SuppressWarnings("Duplicates") 74 @SuppressWarnings("Duplicates")
65 -public class KuaiShouCrawl implements CrawlStrategy { 75 +public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
66 76
  77 + private boolean isRunning = false;
67 private final VideoPoolDao videoPoolDao; 78 private final VideoPoolDao videoPoolDao;
68 private final LivePoolDao livePoolDao; 79 private final LivePoolDao livePoolDao;
69 private final AccountDao accountDao; 80 private final AccountDao accountDao;
@@ -89,25 +100,46 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -89,25 +100,46 @@ public class KuaiShouCrawl implements CrawlStrategy {
89 final List<Account> accountList = accountDao.getAllKSAccount(); 100 final List<Account> accountList = accountDao.getAllKSAccount();
90 accountList.forEach(item -> 101 accountList.forEach(item ->
91 Arrays.stream(DataTypeEnum.values()).forEach(item1 -> 102 Arrays.stream(DataTypeEnum.values()).forEach(item1 ->
92 - threadPoolExecutor.execute(() -> {  
93 - final String accountNo = item.getPhoneNo();  
94 - final Integer typeValue = item1.getValue();  
95 - String key = accountNo + "#" + typeValue;  
96 - final String ns_sig3 = this.getNS_sig3(accountNo, typeValue);  
97 - if (Objects.nonNull(ns_sig3))  
98 - sig3Map.put(key, ns_sig3);  
99 - }) 103 + threadPoolExecutor.execute(() -> this.task(item.getAccountNo(), item1.getValue()))
100 ) 104 )
101 ); 105 );
102 return true; 106 return true;
103 } 107 }
104 108
  109 + /**
  110 + * 获取类型签名task
  111 + *
  112 + * @param accountNo 账户号
  113 + * @param dataType 数据类型
  114 + */
  115 + public void task(String accountNo, Integer dataType) {
  116 + String key = accountNo + "#" + dataType;
  117 + final String ns_sig3 = this.getNS_sig3(accountNo, dataType, true);
  118 + if (Objects.nonNull(ns_sig3))
  119 + sig3Map.put(key, ns_sig3);
  120 + }
  121 +
  122 + /**
  123 + * 停止设置sig3Map线程池
  124 + *
  125 + * @return
  126 + */
105 public boolean stopSetSig3Map() { 127 public boolean stopSetSig3Map() {
106 threadPoolExecutor.shutdownNow(); 128 threadPoolExecutor.shutdownNow();
107 return true; 129 return true;
108 } 130 }
109 131
110 /** 132 /**
  133 + * 清空sig3Map
  134 + *
  135 + * @return
  136 + */
  137 + public boolean cleanSig3Map() {
  138 + sig3Map.clear();
  139 + return true;
  140 + }
  141 +
  142 + /**
111 * 获取sig3Map 143 * 获取sig3Map
112 */ 144 */
113 public String getSig3Map() { 145 public String getSig3Map() {
@@ -117,8 +149,13 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -117,8 +149,13 @@ public class KuaiShouCrawl implements CrawlStrategy {
117 /** 149 /**
118 * 设置sig3Map 150 * 设置sig3Map
119 */ 151 */
120 - public boolean setMapFromString(String jsonStr) {  
121 - HashMap<String, String> map = JSON.parseObject(JSON.parseObject(jsonStr).getString("jsonStr"), new TypeReference<>() { 152 + public boolean setMapFromString(String jsonStr, boolean interior) {
  153 + String jsonString;
  154 + if (interior)
  155 + jsonString = jsonStr;
  156 + else
  157 + jsonString = JSON.parseObject(jsonStr).getString("jsonStr");
  158 + HashMap<String, String> map = JSON.parseObject(jsonString, new TypeReference<>() {
122 }, Feature.OrderedField); 159 }, Feature.OrderedField);
123 map.forEach(sig3Map::put); 160 map.forEach(sig3Map::put);
124 return true; 161 return true;
@@ -215,7 +252,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -215,7 +252,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
215 CookieStore cookieStore = new BasicCookieStore(); 252 CookieStore cookieStore = new BasicCookieStore();
216 cookies.setCookieStore(cookieStore); 253 cookies.setCookieStore(cookieStore);
217 Date previousDay = DateUtil.getPreviousDay(new Date()); 254 Date previousDay = DateUtil.getPreviousDay(new Date());
218 - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue()); 255 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue(), false);
219 if (Objects.isNull(ns_sig3)) 256 if (Objects.isNull(ns_sig3))
220 return null; 257 return null;
221 Map<String, Object> params = new LinkedHashMap<>(); 258 Map<String, Object> params = new LinkedHashMap<>();
@@ -243,8 +280,12 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -243,8 +280,12 @@ public class KuaiShouCrawl implements CrawlStrategy {
243 if (!StringUtils.hasText(res)) { 280 if (!StringUtils.hasText(res)) {
244 throw new BusinessException("调用快手[视频]接口失败"); 281 throw new BusinessException("调用快手[视频]接口失败");
245 } 282 }
  283 + if (Objects.equals(response.getInteger("result"), 500002)) {
  284 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.VIDEO.getValue()));
  285 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  286 + }
246 JSONArray videoJsonArray = Optional.ofNullable(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject()).getJSONArray("photoList")).orElse(new JSONArray()); 287 JSONArray videoJsonArray = Optional.ofNullable(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject()).getJSONArray("photoList")).orElse(new JSONArray());
247 - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue()); 288 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue());
248 // 视频数据存库 289 // 视频数据存库
249 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size()); 290 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size());
250 videoJsonArray.forEach(item -> { 291 videoJsonArray.forEach(item -> {
@@ -286,7 +327,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -286,7 +327,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
286 .playCount(Optional.ofNullable(obj.getInteger("playCount")).orElse(0))// 327 .playCount(Optional.ofNullable(obj.getInteger("playCount")).orElse(0))//
287 .likeCount(Optional.ofNullable(obj.getInteger("likeCount")).orElse(0))// 328 .likeCount(Optional.ofNullable(obj.getInteger("likeCount")).orElse(0))//
288 .commentCount(Optional.ofNullable(obj.getInteger("commentCount")).orElse(0))// 329 .commentCount(Optional.ofNullable(obj.getInteger("commentCount")).orElse(0))//
289 - .phoneNo(accountNo) 330 + .accountNo(accountNo)
290 .reportDate(new Date()) 331 .reportDate(new Date())
291 .publishTime(PublicUtil.parseDate(obj.getString("publishTime")))// 332 .publishTime(PublicUtil.parseDate(obj.getString("publishTime")))//
292 .videoUrl("https://www.kuaishou.com/short-video/" + obj.getString("workId")) 333 .videoUrl("https://www.kuaishou.com/short-video/" + obj.getString("workId"))
@@ -305,7 +346,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -305,7 +346,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
305 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存 346 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
306 } else { 347 } else {
307 final VideoPool nullVideo = VideoPool.builder() 348 final VideoPool nullVideo = VideoPool.builder()
308 - .phoneNo(accountNo) 349 + .accountNo(accountNo)
309 .type(this.getType().getValue()) 350 .type(this.getType().getValue())
310 .resourceType(ResourceTypeEnum.VIDEO.getValue()) 351 .resourceType(ResourceTypeEnum.VIDEO.getValue())
311 .reportDate(new Date()) 352 .reportDate(new Date())
@@ -341,7 +382,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -341,7 +382,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
341 Date endTime = DateUtil.getThisDayMaxTime(previousDay); 382 Date endTime = DateUtil.getThisDayMaxTime(previousDay);
342 Date startTime = DateUtil.getThisDayMinTime(previousDay); 383 Date startTime = DateUtil.getThisDayMinTime(previousDay);
343 //Date startTime = DateUtil.getThisDayMinTime(new Date(previousDay.getTime() - 7 * 24 * 3600 * 1000L));// 补数据使用 384 //Date startTime = DateUtil.getThisDayMinTime(new Date(previousDay.getTime() - 7 * 24 * 3600 * 1000L));// 补数据使用
344 - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue()); 385 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue(), false);
345 if (Objects.isNull(ns_sig3)) 386 if (Objects.isNull(ns_sig3))
346 return null; 387 return null;
347 Map<String, Object> params = new LinkedHashMap<>(); 388 Map<String, Object> params = new LinkedHashMap<>();
@@ -371,6 +412,10 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -371,6 +412,10 @@ public class KuaiShouCrawl implements CrawlStrategy {
371 if (!StringUtils.hasText(res)) { 412 if (!StringUtils.hasText(res)) {
372 throw new BusinessException("调用快手[视频]接口失败"); 413 throw new BusinessException("调用快手[视频]接口失败");
373 } 414 }
  415 + if (Objects.equals(response.getInteger("result"), 500002)) {
  416 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.LIVE.getValue()));
  417 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  418 + }
374 JSONObject dataJSONObject = response.getJSONObject("data"); 419 JSONObject dataJSONObject = response.getJSONObject("data");
375 JSONArray dataJSONArray = dataJSONObject.getJSONArray("details"); 420 JSONArray dataJSONArray = dataJSONObject.getJSONArray("details");
376 /*List<JSONObject> collect = new ArrayList<>(); 421 /*List<JSONObject> collect = new ArrayList<>();
@@ -424,49 +469,9 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -424,49 +469,9 @@ public class KuaiShouCrawl implements CrawlStrategy {
424 * userHead: "https://tx2.a.kwimgs.com/uhead/AB/2021/11/11/12/BMjAyMTExMTExMjUwMzZfMjU2MTc2NDMyMV8xX2hkMjg5XzMyOA==_s.jpg" 469 * userHead: "https://tx2.a.kwimgs.com/uhead/AB/2021/11/11/12/BMjAyMTExMTExMjUwMzZfMjU2MTc2NDMyMV8xX2hkMjg5XzMyOA==_s.jpg"
425 * -userName: "长安汽车。小明聊聊车" 470 * -userName: "长安汽车。小明聊聊车"
426 */ 471 */
427 - /*String coverUrl = obj.getString("liveCover");// 直播封面地址  
428 - String playbackUrl = null;  
429 - final Double liveDuration = obj.getDouble("liveDuration");// 数据接口返回直播时长  
430 - final Long liveStartTimeStamp = obj.getLong("liveTime");// 数据接口返回开播时间戳  
431 - if (liveDuration >= 60) {// 直播时长大于等于60分钟才记录回放信息  
432 - if (!CollectionUtils.isEmpty(finalCollect)) {  
433 - JSONObject playbackMsg;  
434 - if (Objects.equals(finalCollect.size(), 1) && Objects.equals(dataJSONArray.size(), 1)) {  
435 - playbackMsg = finalCollect.get(0);  
436 - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长  
437 - final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳  
438 - final double timeSubAbs = Math.abs(liveDuration - duration);  
439 - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();  
440 - // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等  
441 - // (liveStartSub < 2)->说明开播时间几乎一样  
442 - if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) {  
443 - playbackMsg = null;  
444 - log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo));  
445 - }  
446 - } else {  
447 - List<JSONObject> collect1 = finalCollect.stream().filter(item1 -> {  
448 - Double duration = item1.getDouble("duration");  
449 - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳  
450 - final double timeSubAbs = Math.abs(liveDuration - duration);  
451 - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();  
452 - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10;  
453 - }).collect(Collectors.toList());  
454 - if (Objects.equals(collect1.size(), 1)) {  
455 - playbackMsg = collect1.get(0);  
456 - } else {  
457 - playbackMsg = null;  
458 - log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo));  
459 - }  
460 - }  
461 - if (Objects.nonNull(playbackMsg)) {  
462 - coverUrl = playbackMsg.getString("coverUrl");  
463 - playbackUrl = this.playbackBaseUrl + playbackMsg.getString("productId");  
464 - }  
465 - }  
466 - }*/  
467 livePoolList.add(LivePool.builder() 472 livePoolList.add(LivePool.builder()
468 .type(this.getType().getValue()) 473 .type(this.getType().getValue())
469 - .phoneNo(accountNo) 474 + .accountNo(accountNo)
470 .reportDate(new Date()) 475 .reportDate(new Date())
471 .commentUserCnt(Optional.ofNullable(obj.getInteger("commentUv")).orElse(0)) 476 .commentUserCnt(Optional.ofNullable(obj.getInteger("commentUv")).orElse(0))
472 .consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0)) 477 .consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
@@ -499,7 +504,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -499,7 +504,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
499 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的快手直播数据"); 504 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的快手直播数据");
500 final LivePool nullLive = LivePool.builder() 505 final LivePool nullLive = LivePool.builder()
501 .type(this.getType().getValue()) 506 .type(this.getType().getValue())
502 - .phoneNo(accountNo) 507 + .accountNo(accountNo)
503 .reportDate(new Date()) 508 .reportDate(new Date())
504 .build(); 509 .build();
505 livePoolDao.save(nullLive); 510 livePoolDao.save(nullLive);
@@ -517,7 +522,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -517,7 +522,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
517 object.put("playbackUrl", null); 522 object.put("playbackUrl", null);
518 if (CollectionUtils.isEmpty(collect)) 523 if (CollectionUtils.isEmpty(collect))
519 return object; 524 return object;
520 - final String accountNo = dbLive.getPhoneNo(); 525 + final String accountNo = dbLive.getAccountNo();
521 final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长 526 final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长
522 final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳 527 final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳
523 if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息 528 if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息
@@ -565,8 +570,8 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -565,8 +570,8 @@ public class KuaiShouCrawl implements CrawlStrategy {
565 JSONArray objects = new JSONArray(); 570 JSONArray objects = new JSONArray();
566 if (Objects.isNull(searchKey)) 571 if (Objects.isNull(searchKey))
567 return objects; 572 return objects;
568 - Map<String, Object> params = new HashMap<>();  
569 - Map<String, Object> params1 = new HashMap<>(); 573 + Map<String, Object> params = new LinkedHashMap<>();
  574 + Map<String, Object> params1 = new LinkedHashMap<>();
570 params1.put("principalId", searchKey); 575 params1.put("principalId", searchKey);
571 params1.put("pcursor", ""); 576 params1.put("pcursor", "");
572 params1.put("count", 150); 577 params1.put("count", 150);
@@ -610,17 +615,17 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -610,17 +615,17 @@ public class KuaiShouCrawl implements CrawlStrategy {
610 */ 615 */
611 @Override 616 @Override
612 @Transactional 617 @Transactional
613 - public Integer updateAccountFans(String accountNo) throws IOException {  
614 - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));  
615 - if (Objects.nonNull(hasFoundFansCnt)) {  
616 - //return hasFoundFansCnt; 618 + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException {
  619 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  620 + if (Objects.nonNull(hasFoundAccountMsg)) {
  621 + return hasFoundAccountMsg;
617 } 622 }
618 HttpCookies cookies = HttpCookies.custom(); 623 HttpCookies cookies = HttpCookies.custom();
619 CookieStore cookieStore = new BasicCookieStore(); 624 CookieStore cookieStore = new BasicCookieStore();
620 cookies.setCookieStore(cookieStore); 625 cookies.setCookieStore(cookieStore);
621 Map<String, Object> params = new HashMap<>(); 626 Map<String, Object> params = new HashMap<>();
622 params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo)); 627 params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo));
623 - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue()); 628 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue(), false);
624 if (Objects.isNull(ns_sig3)) 629 if (Objects.isNull(ns_sig3))
625 return null; 630 return null;
626 HttpConfig config = HttpConfig.custom() 631 HttpConfig config = HttpConfig.custom()
@@ -643,24 +648,33 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -643,24 +648,33 @@ public class KuaiShouCrawl implements CrawlStrategy {
643 if (!StringUtils.hasText(res)) { 648 if (!StringUtils.hasText(res)) {
644 throw new BusinessException("调用快手[直播]接口失败"); 649 throw new BusinessException("调用快手[直播]接口失败");
645 } 650 }
646 - Integer fansNum;  
647 - if (Objects.equals(response.getInteger("result"), 500002))  
648 - fansNum = 0;  
649 - else  
650 - fansNum = response.getJSONObject("data").getInteger("fansCnt");  
651 - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue()); 651 + if (Objects.equals(response.getInteger("result"), 500002)) {
  652 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.FANS.getValue()));
  653 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  654 + }
  655 + JSONObject data = response.getJSONObject("data");
  656 + ReportAccountDto reportAccountDto = ReportAccountDto.builder()
  657 + .fansCnt(data.getInteger("fansCnt"))
  658 + .followCnt(data.getInteger("followCnt"))
  659 + .likeCnt(data.getInteger("likeCnt"))
  660 + .accountId(data.getString("userId"))
  661 + .userOtherId(data.getString("userKwaiId"))
  662 + .accountName(data.getString("userName"))
  663 + .build();
  664 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
652 if (Objects.nonNull(account)) 665 if (Objects.nonNull(account))
653 - accountDao.updateFans(account.getId(), fansNum, new Date()); 666 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
654 else 667 else
655 accountDao.save(Account.builder() 668 accountDao.save(Account.builder()
656 .cookiesStatus(true) 669 .cookiesStatus(true)
657 - .phoneNo(accountNo) 670 + .accountNo(accountNo)
658 .type(this.getType().getValue()) 671 .type(this.getType().getValue())
659 - .fansCnt(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt")) 672 + .fansCnt(reportAccountDto.getFansCnt())
  673 + .accountName(reportAccountDto.getAccountName())
660 .reportDate(new Date()) 674 .reportDate(new Date())
661 .done(false) 675 .done(false)
662 .build()); 676 .build());
663 - return Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt"); 677 + return reportAccountDto;
664 } 678 }
665 679
666 /** 680 /**
@@ -700,12 +714,16 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -700,12 +714,16 @@ public class KuaiShouCrawl implements CrawlStrategy {
700 * 714 *
701 * @param accountNo 快手账户号 715 * @param accountNo 快手账户号
702 * @param type 密钥类型(1:粉丝, 2:短视频, 3:直播) 716 * @param type 密钥类型(1:粉丝, 2:短视频, 3:直播)
  717 + * @param retryGet 是否重新获取
703 */ 718 */
704 - public String getNS_sig3(String accountNo, Integer type) { 719 + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) {
705 final String key = accountNo + "#" + type; 720 final String key = accountNo + "#" + type;
706 - String NS_sig3 = sig3Map.get(key);  
707 - if (StringUtils.hasText(NS_sig3))  
708 - return NS_sig3; 721 + String NS_sig3;
  722 + if (!retryGet) {
  723 + NS_sig3 = sig3Map.get(key);
  724 + if (StringUtils.hasText(NS_sig3))
  725 + return NS_sig3;
  726 + }
709 final String uuid = UUID.randomUUID().toString().replace("-", ""); 727 final String uuid = UUID.randomUUID().toString().replace("-", "");
710 final WebDriver driver = this.getKSDriver(accountNo, uuid); 728 final WebDriver driver = this.getKSDriver(accountNo, uuid);
711 String targetUrl = null; 729 String targetUrl = null;
@@ -852,7 +870,7 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -852,7 +870,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
852 * @return 870 * @return
853 */ 871 */
854 public String getRandomUserCookies() { 872 public String getRandomUserCookies() {
855 - final List<FwCookie> cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getPhoneNo(), this.getType().getValue()); 873 + final List<FwCookie> cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getAccountNo(), this.getType().getValue());
856 StringBuffer sb = new StringBuffer(); 874 StringBuffer sb = new StringBuffer();
857 sb.append("clientid=3;"); 875 sb.append("clientid=3;");
858 cookies.forEach(item -> { 876 cookies.forEach(item -> {
@@ -903,4 +921,78 @@ public class KuaiShouCrawl implements CrawlStrategy { @@ -903,4 +921,78 @@ public class KuaiShouCrawl implements CrawlStrategy {
903 return true; 921 return true;
904 } 922 }
905 923
  924 + @Override
  925 + public int getPhase() {
  926 + return 1;// 默认为0
  927 + }
  928 +
  929 + @Override
  930 + public boolean isAutoStartup() {
  931 + return true;// 默认为false
  932 + }
  933 +
  934 + @Override
  935 + public void stop(Runnable callback) {
  936 + log.info("springIOC停止, 将签名信息从Map中导出到配置文件");
  937 + callback.run();
  938 + isRunning = false;
  939 + final HashMap<String, String> props = new HashMap<>();
  940 + props.put("KSAccountNS_sig", JSON.toJSONString(sig3Map));
  941 + updateProperties("NS_sig3Msg.properties", props);
  942 + }
  943 +
  944 + @Override
  945 + public void start() {
  946 + log.info("springIOC启动, 将签名信息从配置文件读出并导入到Map中");
  947 + isRunning = true;
  948 + Properties props;
  949 + try {
  950 + props = PropertiesLoaderUtils.loadAllProperties("NS_sig3Msg.properties");
  951 + for (Object key : props.keySet()) {
  952 + if (Objects.equals(key, "KSAccountNS_sig"))
  953 + this.setMapFromString(props.get(key).toString(), true);
  954 + }
  955 + } catch (IOException e) {
  956 + log.error(e.getMessage());
  957 + }
  958 + }
  959 +
  960 + @Override
  961 + public void stop() {
  962 + log.info("stop");
  963 + isRunning = false;
  964 + }
  965 +
  966 + @Override
  967 + public boolean isRunning() {
  968 + return isRunning;
  969 + }
  970 +
  971 + public void updateProperties(String fileName, Map<String, String> keyValueMap) {
  972 + String filePath = Objects.requireNonNull(PropertiesUtil.class.getClassLoader().getResource(fileName)).getFile();
  973 + Properties props;
  974 + BufferedWriter bw = null;
  975 + try {
  976 + filePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8);
  977 + log.debug("updateProperties propertiesPath:" + filePath);
  978 + props = PropertiesLoaderUtils.loadProperties(new ClassPathResource(fileName));
  979 + log.debug("updateProperties old:" + props);
  980 + bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath)));// 写入属性文件
  981 + //props.clear();// 清空旧的文件
  982 + for (String key : keyValueMap.keySet()) {
  983 + props.setProperty(key, keyValueMap.get(key));
  984 + }
  985 + log.debug("updateProperties new:" + props);
  986 + props.store(bw, "");
  987 + } catch (IOException e) {
  988 + log.error(e.getMessage());
  989 + } finally {
  990 + try {
  991 + assert bw != null;
  992 + bw.close();
  993 + } catch (IOException e) {
  994 + e.printStackTrace();
  995 + }
  996 + }
  997 + }
906 } 998 }
src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java
  View file @d472478
@@ -35,13 +35,13 @@ public class AccountRpcService { @@ -35,13 +35,13 @@ public class AccountRpcService {
35 /** 35 /**
36 * 账号cookie失效, 上报服务器 36 * 账号cookie失效, 上报服务器
37 * 37 *
38 - * @param phoneNo 账号  
39 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili) 38 + * @param accountNo 账号
  39 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
40 * @return 结果 40 * @return 结果
41 */ 41 */
42 - public boolean pushExpireAccount(String phoneNo, Integer type) { 42 + public boolean pushExpireAccount(String accountNo, Integer type) {
43 Map<String, Object> params = new HashMap<>(); 43 Map<String, Object> params = new HashMap<>();
44 - params.put("account", phoneNo); 44 + params.put("account", accountNo);
45 params.put("type", type); 45 params.put("type", type);
46 HttpConfig config = HttpConfig.custom() 46 HttpConfig config = HttpConfig.custom()
47 .url(getBaseUrl() + PathConstant.EXPIRE_ACCOUNT) 47 .url(getBaseUrl() + PathConstant.EXPIRE_ACCOUNT)
@@ -55,7 +55,7 @@ public class AccountRpcService { @@ -55,7 +55,7 @@ public class AccountRpcService {
55 String message = Optional.ofNullable(resObj.getString("message")).orElse("账号cookie失效上报失败"); 55 String message = Optional.ofNullable(resObj.getString("message")).orElse("账号cookie失效上报失败");
56 throw new BusinessException(status, message); 56 throw new BusinessException(status, message);
57 } 57 }
58 - accountService.updateAccountCookiesStatus(phoneNo, type, false); 58 + accountService.updateAccountCookiesStatus(accountNo, type, false);
59 return true; 59 return true;
60 } 60 }
61 61
@@ -86,7 +86,7 @@ public class AccountRpcService { @@ -86,7 +86,7 @@ public class AccountRpcService {
86 jsonArray.forEach(item -> { 86 jsonArray.forEach(item -> {
87 final JSONObject obj = (JSONObject) item; 87 final JSONObject obj = (JSONObject) item;
88 final AccountDto dto = new AccountDto(); 88 final AccountDto dto = new AccountDto();
89 - dto.setPhoneNo(obj.getString("account")); 89 + dto.setAccountNo(obj.getString("account"));
90 dto.setPlaybackSearchKey(obj.getString("playbackSearchKey")); 90 dto.setPlaybackSearchKey(obj.getString("playbackSearchKey"));
91 dto.setGroupId(obj.getLong("groupId")); 91 dto.setGroupId(obj.getLong("groupId"));
92 list.add(dto); 92 list.add(dto);
src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java
  View file @d472478
@@ -4,6 +4,7 @@ import cn.fw.freya.common.PathConstant; @@ -4,6 +4,7 @@ import cn.fw.freya.common.PathConstant;
4 import cn.fw.freya.model.data.pool.LivePool; 4 import cn.fw.freya.model.data.pool.LivePool;
5 import cn.fw.freya.model.data.pool.VideoPool; 5 import cn.fw.freya.model.data.pool.VideoPool;
6 import cn.fw.freya.model.dto.rpc.LivePoolDto; 6 import cn.fw.freya.model.dto.rpc.LivePoolDto;
  7 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
7 import cn.fw.freya.model.dto.rpc.VideoPoolDto; 8 import cn.fw.freya.model.dto.rpc.VideoPoolDto;
8 import cn.fw.freya.utils.RequestUtil; 9 import cn.fw.freya.utils.RequestUtil;
9 import cn.fw.freya.utils.http.HttpConfig; 10 import cn.fw.freya.utils.http.HttpConfig;
@@ -32,18 +33,19 @@ public class ReportRpcService { @@ -32,18 +33,19 @@ public class ReportRpcService {
32 private String baseUrl; 33 private String baseUrl;
33 34
34 /** 35 /**
35 - * 账号粉丝数上报 36 + * 账户信息上报
36 * 37 *
37 - * @param accountNo 账户号  
38 - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)  
39 - * @param fans 粉丝数 38 + * @param accountNo 账户号
  39 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  40 + * @param accountMsg 账户信息
40 * @return 41 * @return
41 */ 42 */
42 - public boolean reportFansCnt(String accountNo, Integer type, Integer fans) { 43 + public boolean reportAccountMsg(String accountNo, Integer type, ReportAccountDto accountMsg) {
43 Map<String, Object> params = new HashMap<>(); 44 Map<String, Object> params = new HashMap<>();
44 params.put("account", accountNo); 45 params.put("account", accountNo);
45 params.put("type", type); 46 params.put("type", type);
46 - params.put("fans", fans); 47 + params.put("fans", accountMsg.getFansCnt());
  48 + params.put("accountName", accountMsg.getAccountName());
47 HttpConfig config = HttpConfig.custom() 49 HttpConfig config = HttpConfig.custom()
48 .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName()) 50 .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName())
49 .url(getBaseUrl() + PathConstant.REPORT_FANS) 51 .url(getBaseUrl() + PathConstant.REPORT_FANS)
src/main/java/cn/fw/freya/task/DataCaptureTask.java
  View file @d472478
@@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject;
16 import lombok.RequiredArgsConstructor; 16 import lombok.RequiredArgsConstructor;
17 import lombok.extern.slf4j.Slf4j; 17 import lombok.extern.slf4j.Slf4j;
18 import org.springframework.context.ApplicationEventPublisher; 18 import org.springframework.context.ApplicationEventPublisher;
19 -import org.springframework.scheduling.annotation.Scheduled;  
20 import org.springframework.stereotype.Component; 19 import org.springframework.stereotype.Component;
21 import org.springframework.util.CollectionUtils; 20 import org.springframework.util.CollectionUtils;
22 21
@@ -51,7 +50,7 @@ public class DataCaptureTask { @@ -51,7 +50,7 @@ public class DataCaptureTask {
51 /** 50 /**
52 * 每分钟执行多线程同时抓取数据 51 * 每分钟执行多线程同时抓取数据
53 */ 52 */
54 - @Scheduled(cron = "0 0/1 * * * ?") 53 + //@Scheduled(cron = "0 0/1 * * * ?")
55 public void capture() { 54 public void capture() {
56 final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); 55 final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool();
57 BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 56 BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列
@@ -78,11 +77,11 @@ public class DataCaptureTask { @@ -78,11 +77,11 @@ public class DataCaptureTask {
78 }); 77 });
79 final Integer accountType = currentAccount.getType();// 当前账户类型 78 final Integer accountType = currentAccount.getType();// 当前账户类型
80 final Integer nextAccountType = Optional.ofNullable(nextAccount.getType()).orElse(-1);// 下一个账户类型 79 final Integer nextAccountType = Optional.ofNullable(nextAccount.getType()).orElse(-1);// 下一个账户类型
81 - final String nextAccountNo = Optional.ofNullable(nextAccount.getPhoneNo()).orElse("");// 下一个账户号 80 + final String nextAccountNo = Optional.ofNullable(nextAccount.getAccountNo()).orElse("");// 下一个账户号
82 final Date todayDayMinTime = DateUtil.getThisDayMinTime(new Date()); 81 final Date todayDayMinTime = DateUtil.getThisDayMinTime(new Date());
83 if (accountType.equals(2) && nextAccountType.equals(2)) { 82 if (accountType.equals(2) && nextAccountType.equals(2)) {
84 if (!Objects.equals(nextAccountNo, "") && !Objects.equals(nextAccountType, -1) && 83 if (!Objects.equals(nextAccountNo, "") && !Objects.equals(nextAccountType, -1) &&
85 - (Objects.nonNull(common.getHasFoundFansCnt(nextAccountNo, 2, todayDayMinTime)) && 84 + (Objects.nonNull(common.getHasFoundAccountMsg(nextAccountNo, 2, todayDayMinTime)) &&
86 Objects.nonNull(common.getHasFoundVideo(nextAccountNo, 2, todayDayMinTime)) && 85 Objects.nonNull(common.getHasFoundVideo(nextAccountNo, 2, todayDayMinTime)) &&
87 Objects.nonNull(common.getHasFoundLive(nextAccountNo, 2, todayDayMinTime)) 86 Objects.nonNull(common.getHasFoundLive(nextAccountNo, 2, todayDayMinTime))
88 )) { 87 )) {
@@ -97,7 +96,7 @@ public class DataCaptureTask { @@ -97,7 +96,7 @@ public class DataCaptureTask {
97 /** 96 /**
98 * 每2分钟执行抓取数据 97 * 每2分钟执行抓取数据
99 */ 98 */
100 - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) 99 + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
101 public void captureLivePlayback() { 100 public void captureLivePlayback() {
102 Double durationThreshold = 60d;// 设置直播时长阈值 101 Double durationThreshold = 60d;// 设置直播时长阈值
103 final Random random = new Random(); 102 final Random random = new Random();
@@ -107,7 +106,7 @@ public class DataCaptureTask { @@ -107,7 +106,7 @@ public class DataCaptureTask {
107 .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据 106 .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据
108 Collection<List<LivePool>> values = withoutPlaybackLive 107 Collection<List<LivePool>> values = withoutPlaybackLive
109 .stream() 108 .stream()
110 - .collect(Collectors.groupingBy(LivePool::getPhoneNo)) 109 + .collect(Collectors.groupingBy(LivePool::getAccountNo))
111 .values() 110 .values()
112 .stream() 111 .stream()
113 .peek(item -> item.sort(Comparator.comparing(LivePool::getGetPlaybackFailTimes)))// 把某个人每条直播按失败次数排序 112 .peek(item -> item.sort(Comparator.comparing(LivePool::getGetPlaybackFailTimes)))// 把某个人每条直播按失败次数排序
@@ -115,8 +114,8 @@ public class DataCaptureTask { @@ -115,8 +114,8 @@ public class DataCaptureTask {
115 .limit(5)// 一次找5个人的直播 114 .limit(5)// 一次找5个人的直播
116 .collect(Collectors.toList()); 115 .collect(Collectors.toList());
117 for (List<LivePool> list : values) {// 遍历每个人的直播集合 116 for (List<LivePool> list : values) {// 遍历每个人的直播集合
118 - final String accountNo = list.get(0).getPhoneNo();  
119 - Account account = accountDao.findByPhoneNoAndType(accountNo, 1);// 获取账号实体 117 + final String accountNo = list.get(0).getAccountNo();
  118 + Account account = accountDao.findByAccountNoAndType(accountNo, 1);// 获取账号实体
120 final List<JSONObject> playbackMsg = this.getPlaybackMsg(account);// 获取该人的直播回放信息 119 final List<JSONObject> playbackMsg = this.getPlaybackMsg(account);// 获取该人的直播回放信息
121 boolean flag = false; 120 boolean flag = false;
122 for (LivePool item : list) { 121 for (LivePool item : list) {
@@ -170,7 +169,7 @@ public class DataCaptureTask { @@ -170,7 +169,7 @@ public class DataCaptureTask {
170 if (Objects.nonNull(playbackSearchKey)) 169 if (Objects.nonNull(playbackSearchKey))
171 userLivePlayback = kuaiShouCrawl.getUserLivePlayback(playbackSearchKey); 170 userLivePlayback = kuaiShouCrawl.getUserLivePlayback(playbackSearchKey);
172 else 171 else
173 - userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getPhoneNo()); 172 + userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getAccountNo());
174 final List<JSONObject> collect = userLivePlayback 173 final List<JSONObject> collect = userLivePlayback
175 .stream() 174 .stream()
176 .filter(item -> { 175 .filter(item -> {
src/main/java/cn/fw/freya/utils/RequestUtil.java
  View file @d472478
@@ -696,10 +696,7 @@ public class RequestUtil { @@ -696,10 +696,7 @@ public class RequestUtil {
696 // 设置通用的请求属性 696 // 设置通用的请求属性
697 conn.setRequestProperty("accept", "*/*"); 697 conn.setRequestProperty("accept", "*/*");
698 conn.setRequestProperty("connection", "Keep-Alive"); 698 conn.setRequestProperty("connection", "Keep-Alive");
699 - conn.setRequestProperty("Host", "cp.kuaishou.com");  
700 conn.setRequestProperty("Content-Type", "application/json"); 699 conn.setRequestProperty("Content-Type", "application/json");
701 - conn.setRequestProperty("Content-Length", "84");  
702 - conn.setRequestProperty("Cookie", "kuaishou.web.cp.api_ph=c634819dfa9a1da1762bafcf5d195fd101b4;kuaishou.web.cp.api_st=ChZrdWFpc2hvdS53ZWIuY3AuYXBpLnN0EqAB_xuzZVzMjD9DNrreTtCT1cQDRKvB60dNkgWBOQI3mk8_F0WAM1qRKnUtcZYuGH0JTMUntF7JMrs8L7vYLP9tpQ25ppobabbfAcPBqztI7OJdc7MqSwrjanYe6PRZCHnU-tALse-CVxzMFtCl304ekWU1o9fjm7VBQ9EEkC9Q35erQ2RbrnucqQI5E3p_9cd0JE9J8odcLgXOjbWQvT0OuhoSPRRvGtXySX7EnaufBy__uLw_IiA-3ieQkuU_bNGof-EIcBSlmCzad9Ut9hWRS5nEILA9FCgFMAE;_did=web_656080669F3F2217;userId=2079628287;did=web_94f084024029be79aade1f079e2b82e1731a;");  
703 // 发送POST请求必须设置如下两行 700 // 发送POST请求必须设置如下两行
704 conn.setDoOutput(true); 701 conn.setDoOutput(true);
705 conn.setDoInput(true); 702 conn.setDoInput(true);
src/main/resources/NS_sig3Msg.properties 0 → 100644
  View file @d472478
  1 +#
  2 +#Thu Jan 27 15:59:12 CST 2022
  3 +KSAccountNS_sig={}
src/main/resources/alert.sql 0 → 100644
  View file @d472478
  1 +alter table ACCOUNT drop constraint UKB04BR4OH2RS73RNUSEXU2NLWG;
  2 +alter table ACCOUNT drop constraint UKSOWL8PT6SDTPQPVTXFDP0NPU0;
  3 +alter table ACCOUNT drop column ACCOUNT_NO;
  4 +alter table ACCOUNT alter column PHONE_NO rename to ACCOUNT_NO;
  5 +
  6 +drop index IDXTGFKFY1N98UJQVJ5ME0VR3FSV;
  7 +drop index IDXKWOBILLHNTYYT02FVAQAM5G5;
  8 +alter table COOKIE drop column ACCOUNT_NO;
  9 +alter table COOKIE alter column PHONE_NO rename to ACCOUNT_NO;
  10 +
  11 +alter table LIVE_POOL drop constraint UKANITO7VXALLS54057JU6KJHDF;
  12 +alter table LIVE_POOL drop constraint UKRQ8QS6Q2H9NPW3GV40B86LRXM;
  13 +alter table LIVE_POOL drop column ACCOUNT_NO;
  14 +alter table LIVE_POOL alter column PHONE_NO rename to ACCOUNT_NO;
  15 +
  16 +alter table VIDEO_POOL drop constraint UKL4792GABX9U74HBTHQ0VQM1FG;
  17 +alter table VIDEO_POOL drop constraint UKQ3ROPUUMLOIHN45FGCSUK77G;
  18 +alter table VIDEO_POOL drop column ACCOUNT_NO;
  19 +alter table VIDEO_POOL alter column PHONE_NO rename to ACCOUNT_NO;