third / fw-freya

Commit d472478d333c0cbbca7312129a8834d36de1962c

Authored by 王明元
1 parent e17d2707

2022年6月30日18:19:28 调用接口时候发现500002错误码, 直接更新sig信息

Inline Side-by-side
Showing 31 changed files with 520 additions and 315 deletions
src/main/java/cn/fw/freya/controller/BLBLController.java
  View file @d472478
... ... @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2  
3 3 import cn.fw.freya.model.data.pool.LivePool;
4 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 6 import cn.fw.freya.service.crawl.impl.BilibiliCrawl;
6 7 import lombok.RequiredArgsConstructor;
7 8 import org.springframework.validation.annotation.Validated;
... ... @@ -52,8 +53,8 @@ public class BLBLController {
52 53 * @param accountNo 账户号
53 54 */
54 55 @GetMapping("/updateAccountFans")
55   - public Integer updateAccountFans(String accountNo) throws IOException {
56   - return bilibiliCrawl.updateAccountFans(accountNo);
  56 + public ReportAccountDto updateAccountFans(String accountNo) throws IOException {
  57 + return bilibiliCrawl.updateAccountMsg(accountNo);
57 58 }
58 59  
59 60 /**
... ...
src/main/java/cn/fw/freya/controller/DCDController.java
  View file @d472478
... ... @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2  
3 3 import cn.fw.freya.model.data.pool.LivePool;
4 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 6 import cn.fw.freya.service.crawl.impl.DongCheDiCrawl;
6 7 import lombok.RequiredArgsConstructor;
7 8 import org.springframework.validation.annotation.Validated;
... ... @@ -61,8 +62,8 @@ public class DCDController {
61 62 * @param accountNo 账户号
62 63 */
63 64 @GetMapping("/updateAccountFans")
64   - public Integer updateAccountFans(String accountNo) {
65   - return dongCheDiCrawl.updateAccountFans(accountNo);
  65 + public ReportAccountDto updateAccountFans(String accountNo) {
  66 + return dongCheDiCrawl.updateAccountMsg(accountNo);
66 67 }
67 68  
68 69 /**
... ...
src/main/java/cn/fw/freya/controller/DYController.java
  View file @d472478
... ... @@ -3,6 +3,7 @@ package cn.fw.freya.controller;
3 3 import cn.fw.freya.model.data.LiveOverview;
4 4 import cn.fw.freya.model.data.pool.LivePool;
5 5 import cn.fw.freya.model.data.pool.VideoPool;
  6 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
6 7 import cn.fw.freya.service.crawl.impl.DouYinCrawl;
7 8 import lombok.RequiredArgsConstructor;
8 9 import org.springframework.validation.annotation.Validated;
... ... @@ -62,8 +63,8 @@ public class DYController {
62 63 * @param accountNo 账户号
63 64 */
64 65 @GetMapping("/updateAccountFans")
65   - public Integer updateAccountFans(String accountNo) {
66   - return douyinCrawl.updateAccountFans(accountNo);
  66 + public ReportAccountDto updateAccountFans(String accountNo) {
  67 + return douyinCrawl.updateAccountMsg(accountNo);
67 68 }
68 69  
69 70 /**
... ...
src/main/java/cn/fw/freya/controller/KSController.java
  View file @d472478
... ... @@ -2,6 +2,7 @@ package cn.fw.freya.controller;
2 2  
3 3 import cn.fw.freya.model.data.pool.LivePool;
4 4 import cn.fw.freya.model.data.pool.VideoPool;
  5 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
5 6 import cn.fw.freya.service.crawl.impl.KuaiShouCrawl;
6 7 import com.alibaba.fastjson.JSONObject;
7 8 import lombok.RequiredArgsConstructor;
... ... @@ -59,8 +60,8 @@ public class KSController {
59 60 * @param accountNo 账户号
60 61 */
61 62 @GetMapping("/updateAccountFans")
62   - public Integer updateAccountFans(String accountNo) throws IOException {
63   - return kuaishouCrawl.updateAccountFans(accountNo);
  63 + public ReportAccountDto updateAccountFans(String accountNo) throws IOException {
  64 + return kuaishouCrawl.updateAccountMsg(accountNo);
64 65 }
65 66  
66 67 /**
... ... @@ -79,13 +80,12 @@ public class KSController {
79 80 * @param accountNo 账户号
80 81 */
81 82 @GetMapping("/getNS_sig3")
82   - public String getNS_sig3(String accountNo, Integer type) {
83   - return kuaishouCrawl.getNS_sig3(accountNo, type);
  83 + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) {
  84 + return kuaishouCrawl.getNS_sig3(accountNo, type, retryGet);
84 85 }
85 86  
86 87 /**
87 88 * 设置setSig3Map
88   - *
89 89 */
90 90 @GetMapping("/setSig3Map")
91 91 public boolean setSig3Map() {
... ... @@ -94,16 +94,14 @@ public class KSController {
94 94  
95 95 /**
96 96 * 设置setSig3Map
97   - *
98 97 */
99 98 @PostMapping("/setMapFromString")
100 99 public boolean setMapFromString(@RequestBody String jsonStr) {
101   - return kuaishouCrawl.setMapFromString(jsonStr);
  100 + return kuaishouCrawl.setMapFromString(jsonStr, false);
102 101 }
103 102  
104 103 /**
105 104 * 停止设置setSig3Map
106   - *
107 105 */
108 106 @GetMapping("/stopSetSig3Map")
109 107 public boolean stopSetSig3Map() {
... ... @@ -111,8 +109,15 @@ public class KSController {
111 109 }
112 110  
113 111 /**
  112 + * 清空Sig3Map
  113 + */
  114 + @GetMapping("/cleanSig3Map")
  115 + public boolean cleanSig3Map() {
  116 + return kuaishouCrawl.cleanSig3Map();
  117 + }
  118 +
  119 + /**
114 120 * 获取getSig3Map
115   - *
116 121 */
117 122 @GetMapping("/getSig3Map")
118 123 public String getSig3Map() {
... ...
src/main/java/cn/fw/freya/controller/OtherController.java
  View file @d472478
... ... @@ -5,6 +5,7 @@ import cn.fw.freya.model.data.Account;
5 5 import cn.fw.freya.model.data.pool.LivePool;
6 6 import cn.fw.freya.model.dto.other.ReportLiveDto;
7 7 import cn.fw.freya.model.dto.other.ReportVideoDto;
  8 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
8 9 import cn.fw.freya.service.CommonBizService;
9 10 import cn.fw.freya.service.crawl.impl.Common;
10 11 import cn.fw.freya.service.rpc.AccountRpcService;
... ... @@ -86,16 +87,16 @@ public class OtherController {
86 87 }
87 88  
88 89 /**
89   - * 上报账户粉丝数
  90 + * 上报账户信息
90 91 *
91   - * @param account 账号
92   - * @param type 账号类型
93   - * @param fansCnt 粉丝数
  92 + * @param account 账号
  93 + * @param type 账号类型
  94 + * @param accountMsg 账户信息
94 95 * @return
95 96 */
96   - @PostMapping("/reportFansCnt")
97   - public ResponseMessage<Boolean> reportFansCnt(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "粉丝数量不能为空") Integer fansCnt) {
98   - return ResponseMessage.success(reportRpcService.reportFansCnt(account, type, fansCnt));
  97 + @PostMapping("/reportAccountMsg")
  98 + public ResponseMessage<Boolean> reportAccountMsg(@NotBlank(message = "账号不能为空") String account, @NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "账户信息不能为空") ReportAccountDto accountMsg) {
  99 + return ResponseMessage.success(reportRpcService.reportAccountMsg(account, type, accountMsg));
99 100 }
100 101  
101 102 /**
... ... @@ -175,7 +176,7 @@ public class OtherController {
175 176 * @return
176 177 */
177 178 @GetMapping("/getWithoutPlaybackLive")
178   - public List<LivePool> getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空")Double durationThreshold) {
  179 + public List<LivePool> getWithoutPlaybackLive(@NotNull(message = "账户类型不能为空") Integer type, @NotNull(message = "时长阈值不能为空") Double durationThreshold) {
179 180 return common.getWithoutPlaybackLive(type, durationThreshold);
180 181 }
181 182  
... ...
src/main/java/cn/fw/freya/dao/AccountDao.java
  View file @d472478
... ... @@ -18,23 +18,23 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; {
18 18 /**
19 19 * 删除指定账号
20 20 *
21   - * @param phoneNo
  21 + * @param accountNo
22 22 * @param value
23 23 */
24 24 @Transactional(rollbackFor = Exception.class)
25 25 @Modifying
26   - @Query("delete from Account a where a.phoneNo = ?1 and a.type = ?2")
27   - void deleteByPhoneNoAndType(String phoneNo, Integer value);
  26 + @Query("delete from Account a where a.accountNo = ?1 and a.type = ?2")
  27 + void deleteByAccountNoAndType(String accountNo, Integer value);
28 28  
29 29 /**
30 30 * 查询账号
31 31 *
32   - * @param phoneNo
  32 + * @param accountNo
33 33 * @param type
34 34 * @return
35 35 */
36   - @Query("select a from Account a where a.phoneNo = ?1 and a.type = ?2 ")
37   - Account findByPhoneNoAndType(String phoneNo, Integer type);
  36 + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 ")
  37 + Account findByAccountNoAndType(String accountNo, Integer type);
38 38  
39 39 /**
40 40 * 随机获取一个账号
... ... @@ -72,45 +72,47 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; {
72 72 List<Account> findNotUseAccount();
73 73  
74 74 /**
75   - * 更新账户粉丝数
  75 + * 更新账户信息
76 76 *
77   - * @param id
78   - * @param fansCnt
  77 + * @param id 账户id
  78 + * @param accountName 账户名
  79 + * @param fansCnt 账户粉丝数
79 80 * @return
80 81 */
81 82 @Transactional(rollbackFor = Exception.class)
82 83 @Modifying
83   - @Query("update Account a set a.fansCnt = ?2, a.reportDate = ?3 where a.id = ?1")
84   - int updateFans(Long id, int fansCnt, Date reportDate);
  84 + @Query("update Account a set a.accountName = ?2, a.fansCnt = ?3, a.reportDate = ?4 where a.id = ?1")
  85 + int updateMsg(Long id, String accountName, int fansCnt, Date reportDate);
85 86  
86 87 /**
87 88 * 获取今天上报的粉丝数据
88 89 *
89   - * @param phoneNo 账户号
  90 + * @param accountNo 账户号
90 91 * @param type 账户类型
91 92 * @param reportDate 上报日期
92 93 * @return
93 94 */
94   - @Query("select a.fansCnt from Account a where a.phoneNo = ?1 and a.type = ?2 and a.reportDate >= ?3")
95   - Integer getHasReportDate(String phoneNo, Integer type, Date reportDate);
  95 + @Query("select a from Account a where a.accountNo = ?1 and a.type = ?2 and a.reportDate >= ?3")
  96 + List<Account> getHasReportDate(String accountNo, Integer type, Date reportDate);
96 97  
97 98 /**
98 99 * 更新账户cookies状态
99 100 *
100   - * @param phoneNo
  101 + * @param accountNo
101 102 * @param type
102 103 * @param cookiesStatus
103 104 */
104 105 @Transactional(rollbackFor = Exception.class)
105 106 @Modifying
106   - @Query("update Account a set a.cookiesStatus = ?3 where a.phoneNo = ?1 and a.type = ?2")
107   - void updateAccountCookiesStatus(String phoneNo, Integer type, boolean cookiesStatus);
  107 + @Query("update Account a set a.cookiesStatus = ?3 where a.accountNo = ?1 and a.type = ?2")
  108 + void updateAccountCookiesStatus(String accountNo, Integer type, boolean cookiesStatus);
108 109  
109 110 /**
110 111 * 设置账户状态为未完成
  112 + *
111 113 * @param accountNo
112 114 */
113 115 @Modifying
114   - @Query("update Account a set a.done = false where a.phoneNo = ?1")
  116 + @Query("update Account a set a.done = false where a.accountNo = ?1")
115 117 void setAccountUndone(String accountNo);
116 118 }
... ...
src/main/java/cn/fw/freya/dao/CookieDao.java
  View file @d472478
... ... @@ -17,21 +17,21 @@ public interface CookieDao extends JpaRepository&lt;FwCookie, Long&gt; {
17 17 /**
18 18 * 删除账号对应的cookie
19 19 *
20   - * @param phoneNo 账户号
21   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  20 + * @param accountNo 账户号
  21 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
22 22 */
23 23 @Transactional
24 24 @Modifying
25   - @Query("delete from FwCookie f where f.phoneNo = ?1 and f.type = ?2")
26   - void deleteByPhoneNoAndType(String phoneNo, Integer type);
  25 + @Query("delete from FwCookie f where f.accountNo = ?1 and f.type = ?2")
  26 + void deleteByAccountNoAndType(String accountNo, Integer type);
27 27  
28 28 /**
29 29 * 查询账号对应平台的cookie
30 30 *
31   - * @param phoneNo 账户号
32   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  31 + * @param accountNo 账户号
  32 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
33 33 * @return
34 34 */
35   - @Query("select f from FwCookie f where f.phoneNo = ?1 and f.type = ?2")
36   - List<FwCookie> findByPhoneNoAndType(String phoneNo, Integer type);
  35 + @Query("select f from FwCookie f where f.accountNo = ?1 and f.type = ?2")
  36 + List<FwCookie> findByAccountNoAndType(String accountNo, Integer type);
37 37 }
... ...
src/main/java/cn/fw/freya/dao/LivePoolDao.java
  View file @d472478
... ... @@ -17,13 +17,13 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; {
17 17 /**
18 18 * 获取今天上报的数据
19 19 *
20   - * @param phoneNo 账户号
  20 + * @param accountNo 账户号
21 21 * @param type 账户类型
22 22 * @param reportDate 上报日期
23 23 * @return
24 24 */
25   - @Query("select live from LivePool live where live.phoneNo = ?1 and live.type = ?2 and live.reportDate >= ?3")
26   - List<LivePool> getHasReportDate(String phoneNo, Integer type, Date reportDate);
  25 + @Query("select live from LivePool live where live.accountNo = ?1 and live.type = ?2 and live.reportDate >= ?3")
  26 + List<LivePool> getHasReportData(String accountNo, Integer type, Date reportDate);
27 27  
28 28 /**
29 29 * 获取没有拿到回播的直播
... ... @@ -41,6 +41,6 @@ public interface LivePoolDao extends JpaRepository&lt;LivePool, Long&gt; {
41 41 * @param accountNo 账户号
42 42 * @return
43 43 */
44   - @Query("select live from LivePool live where live.phoneNo = ?1")
  44 + @Query("select live from LivePool live where live.accountNo = ?1")
45 45 List<LivePool> getLiveByAccountNo(String accountNo);
46 46 }
... ...
src/main/java/cn/fw/freya/dao/VideoPoolDao.java
  View file @d472478
... ... @@ -21,24 +21,24 @@ public interface VideoPoolDao extends JpaRepository&lt;VideoPool, Long&gt; {
21 21 /**
22 22 * 删除
23 23 *
24   - * @param phoneNo 账户号
  24 + * @param accountNo 账户号
25 25 * @param previousDay 前一天
26 26 * @param type 账户类型
27 27 * @param resourceType 资源类型
28 28 */
29 29 @Transactional(rollbackFor = Exception.class)
30 30 @Modifying
31   - @Query("delete from VideoPool where phoneNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4")
32   - void deleteByPhoneNoAndDate(String phoneNo, Date previousDay, Integer type, Integer resourceType);
  31 + @Query("delete from VideoPool where accountNo = ?1 and reportDate = ?2 and type=?3 and resourceType=?4")
  32 + void deleteByAccountNoAndDate(String accountNo, Date previousDay, Integer type, Integer resourceType);
33 33  
34 34 /**
35 35 * 获取今天上报的数据
36 36 *
37   - * @param phoneNo 账户号
  37 + * @param accountNo 账户号
38 38 * @param type 账户类型
39 39 * @param reportDate 上报日期
40 40 * @return
41 41 */
42   - @Query("select v from VideoPool v where v.phoneNo = ?1 and v.type = ?2 and v.reportDate >= ?3")
43   - List<VideoPool> getHasReportDate(String phoneNo, Integer type, Date reportDate);
  42 + @Query("select v from VideoPool v where v.accountNo = ?1 and v.type = ?2 and v.reportDate >= ?3")
  43 + List<VideoPool> getHasReportDate(String accountNo, Integer type, Date reportDate);
44 44 }
... ...
src/main/java/cn/fw/freya/enums/AccountTypeEnum.java
  View file @d472478
... ... @@ -11,7 +11,7 @@ public enum AccountTypeEnum {
11 11 KS(1, "快手"),// 快手
12 12 DY(2, "抖音"),// 抖音
13 13 DCD(3, "懂车帝"),// 懂车帝
14   - BILIBILI(4, "哔哩哔哩动画"),// 哔哩哔哩动画
  14 + BILIBILI(4, "哔哩哔哩"),// 哔哩哔哩
15 15 ;
16 16 @Getter
17 17 private final Integer value;
... ...
src/main/java/cn/fw/freya/model/data/Account.java
  View file @d472478
... ... @@ -18,16 +18,12 @@ import java.util.Objects;
18 18 @AllArgsConstructor
19 19 @Builder(toBuilder = true)
20 20 @Entity
21   -@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "type"}))
  21 +@Table(name = "account", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type"}))
22 22 public class Account {
23 23 @Id
24 24 @GeneratedValue(strategy = GenerationType.IDENTITY)
25 25 private Long id;
26 26 /**
27   - * 账户号
28   - */
29   - private String phoneNo;
30   - /**
31 27 * 回播搜索关键词
32 28 */
33 29 private String playbackSearchKey;
... ... @@ -36,6 +32,10 @@ public class Account {
36 32 */
37 33 private String accountNo;
38 34 /**
  35 + * 账户名
  36 + */
  37 + private String accountName;
  38 + /**
39 39 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
40 40 */
41 41 private Integer type;
... ...
src/main/java/cn/fw/freya/model/data/FwCookie.java
  View file @d472478
... ... @@ -23,7 +23,7 @@ import java.util.Objects;
23 23 @ToString
24 24 @AllArgsConstructor
25 25 @NoArgsConstructor
26   -@Table(name = "cookie", indexes = {@Index(columnList = "phoneNo")})
  26 +@Table(name = "cookie", indexes = {@Index(columnList = "accountNo")})
27 27 public class FwCookie {
28 28 @Id
29 29 @GeneratedValue(strategy = GenerationType.IDENTITY)
... ... @@ -31,7 +31,7 @@ public class FwCookie {
31 31 /**
32 32 * 账户号
33 33 */
34   - private String phoneNo;
  34 + private String accountNo;
35 35 /**
36 36 * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
37 37 */
... ... @@ -104,13 +104,13 @@ public class FwCookie {
104 104 /**
105 105 * WebDriver获取到的cookies->DB
106 106 *
107   - * @param cookie WebDriver获取到的cookie
108   - * @param phoneNo 账户号
  107 + * @param cookie WebDriver获取到的cookie
  108 + * @param accountNo 账户号
109 109 * @return cookies
110 110 */
111   - public static FwCookie toDb(Cookie cookie, String phoneNo, Integer type) {
  111 + public static FwCookie toDb(Cookie cookie, String accountNo, Integer type) {
112 112 FwCookie fwCookie = new FwCookie();
113   - fwCookie.setPhoneNo(phoneNo);
  113 + fwCookie.setAccountNo(accountNo);
114 114 fwCookie.setType(type);
115 115 fwCookie.setDomain(cookie.getDomain());
116 116 fwCookie.setExpiryDate(DateUtil.date2LocalDateTime(cookie.getExpiry()));
... ...
src/main/java/cn/fw/freya/model/data/LiveOverview.java
  View file @d472478
... ... @@ -18,7 +18,7 @@ import java.util.Date;
18 18 @Builder(toBuilder = true)
19 19 @Entity
20 20 @ToString
21   -@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "reportDate", "type"}))
  21 +@Table(name = "live_overview", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "reportDate", "type"}))
22 22 public class LiveOverview {
23 23 @Id
24 24 @GeneratedValue(strategy = GenerationType.IDENTITY)
... ... @@ -41,7 +41,7 @@ public class LiveOverview {
41 41 /**
42 42 * 账户号
43 43 */
44   - private String phoneNo;
  44 + private String accountNo;
45 45 /**
46 46 * 直播场次
47 47 */
... ...
src/main/java/cn/fw/freya/model/data/pool/LivePool.java
  View file @d472478
... ... @@ -20,7 +20,7 @@ import java.util.Objects;
20 20 @AllArgsConstructor
21 21 @Builder(toBuilder = true)
22 22 @Entity
23   -@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "phoneNo", "roomId"}))
  23 +@Table(name = "live_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"reportDate", "type", "accountNo", "roomId"}))
24 24 public class LivePool {
25 25 /**
26 26 * 主键id
... ... @@ -35,7 +35,7 @@ public class LivePool {
35 35 /**
36 36 * 账户号
37 37 */
38   - private String phoneNo;
  38 + private String accountNo;
39 39 /**
40 40 * 数据上报日期
41 41 */
... ...
src/main/java/cn/fw/freya/model/data/pool/VideoPool.java
  View file @d472478
... ... @@ -19,7 +19,7 @@ import java.util.Date;
19 19 @Builder(toBuilder = true)
20 20 @Entity
21 21 @ToString
22   -@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"phoneNo", "videoId", "type", "reportDate"}))
  22 +@Table(name = "video_pool", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "videoId", "type", "reportDate"}))
23 23 public class VideoPool {
24 24 @Id
25 25 @GeneratedValue(strategy = GenerationType.IDENTITY)
... ... @@ -27,7 +27,7 @@ public class VideoPool {
27 27 /**
28 28 * 账户号
29 29 */
30   - private String phoneNo;
  30 + private String accountNo;
31 31 /**
32 32 * 视频id
33 33 **/
... ...
src/main/java/cn/fw/freya/model/dto/rpc/AccountDto.java
  View file @d472478
... ... @@ -12,7 +12,7 @@ public class AccountDto {
12 12 /**
13 13 * 账号
14 14 */
15   - private String phoneNo;
  15 + private String accountNo;
16 16 /**
17 17 * 回播搜索关键词
18 18 */
... ...
src/main/java/cn/fw/freya/model/dto/rpc/ReportAccountDto.java 0 → 100644
  View file @d472478
  1 +package cn.fw.freya.model.dto.rpc;
  2 +
  3 +import lombok.AllArgsConstructor;
  4 +import lombok.Builder;
  5 +import lombok.Data;
  6 +import lombok.NoArgsConstructor;
  7 +
  8 +/**
  9 + * @author unknown
  10 + * @version 1.0
  11 + * @date 2022/6/29 16:10
  12 + * @Description
  13 + */
  14 +@Data
  15 +@NoArgsConstructor
  16 +@AllArgsConstructor
  17 +@Builder(toBuilder = true)
  18 +public class ReportAccountDto {
  19 + /**
  20 + * 粉丝数
  21 + */
  22 + private Integer fansCnt;
  23 + /**
  24 + * 关注数
  25 + */
  26 + private Integer followCnt;
  27 + /**
  28 + * 点赞数
  29 + */
  30 + private Integer likeCnt;
  31 + /**
  32 + * 用户在平台的其他信息id
  33 + */
  34 + private String userOtherId;
  35 + /**
  36 + * 用户id
  37 + */
  38 + private String accountId;
  39 + /**
  40 + * 用户名
  41 + */
  42 + private String accountName;
  43 +}
... ...
src/main/java/cn/fw/freya/service/CommonBizService.java
  View file @d472478
... ... @@ -113,7 +113,7 @@ public class CommonBizService {
113 113 }
114 114 accountList.addAll(list.stream()
115 115 .map(item -> Account.builder()
116   - .phoneNo(item.getPhoneNo())
  116 + .accountNo(item.getAccountNo())
117 117 .playbackSearchKey(item.getPlaybackSearchKey())
118 118 .type(typeEnum.getValue())
119 119 .cookiesStatus(Boolean.TRUE)
... ...
src/main/java/cn/fw/freya/service/CrawlBizService.java
  View file @d472478
... ... @@ -4,6 +4,7 @@ import cn.fw.freya.enums.AccountTypeEnum;
4 4 import cn.fw.freya.model.data.Account;
5 5 import cn.fw.freya.model.data.pool.LivePool;
6 6 import cn.fw.freya.model.data.pool.VideoPool;
  7 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
7 8 import cn.fw.freya.service.crawl.CrawlStrategy;
8 9 import cn.fw.freya.service.data.AccountService;
9 10 import cn.fw.freya.service.rpc.AccountRpcService;
... ... @@ -53,43 +54,43 @@ public class CrawlBizService {
53 54 /**
54 55 * 登陆准备
55 56 *
56   - * @param phoneNo 账户号
57   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  57 + * @param accountNo 账户号
  58 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
58 59 * @return
59 60 */
60   - public String preLogin(String phoneNo, Integer type) {
  61 + public String preLogin(String accountNo, Integer type) {
61 62 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
62 63 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
63 64 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
64   - return crawlStrategy.preLogin(phoneNo);
  65 + return crawlStrategy.preLogin(accountNo);
65 66 }
66 67  
67 68 /**
68 69 * 登陆
69 70 *
70   - * @param phoneNo 账户号
71   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  71 + * @param accountNo 账户号
  72 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
72 73 * @return
73 74 */
74   - public boolean doLogin(String phoneNo, Integer type) {
  75 + public boolean doLogin(String accountNo, Integer type) {
75 76 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
76 77 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
77 78 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
78   - return crawlStrategy.doLogin(phoneNo);
  79 + return crawlStrategy.doLogin(accountNo);
79 80 }
80 81  
81 82 /**
82 83 * 退出浏览器
83 84 *
84   - * @param phoneNo 账户号
85   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  85 + * @param accountNo 账户号
  86 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
86 87 * @return
87 88 */
88   - public boolean exitBrowser(String phoneNo, Integer type) {
  89 + public boolean exitBrowser(String accountNo, Integer type) {
89 90 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
90 91 AssertUtil.notNull(typeEnum, () -> "平台类型不正确");
91 92 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
92   - return crawlStrategy.exitBrowser(phoneNo);
  93 + return crawlStrategy.exitBrowser(accountNo);
93 94 }
94 95  
95 96 /**
... ... @@ -99,13 +100,13 @@ public class CrawlBizService {
99 100 */
100 101 public void crawlData(Account account) throws IOException {
101 102 Integer type = account.getType();
102   - final String accountNo = account.getPhoneNo();
  103 + final String accountNo = account.getAccountNo();
103 104 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
104 105 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
105 106 // 抓取数据
106 107 log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据");
107   - final Integer fansCnt = crawlStrategy.updateAccountFans(accountNo);// 更新粉丝数
108   - if (Objects.isNull(fansCnt)) {
  108 + final ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数
  109 + if (Objects.isNull(accountMsg)) {
109 110 doPushExpireAccount(account);
110 111 return;
111 112 }
... ... @@ -120,19 +121,20 @@ public class CrawlBizService {
120 121 return;
121 122 }
122 123 // 上报数据
123   - final boolean reportFansCnt = this.doReportFansCnt(account, fansCnt);
124   - if (!reportFansCnt) {
125   - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的粉丝数据失败");
  124 + final boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg);
  125 + final String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo);
  126 + if (!reportAccountMsg) {
  127 + log.error(format + "的账户信息失败");
126 128 return;
127 129 }
128 130 final boolean reportVideo = this.doReportVideo(account, allVideoMsg);
129 131 if (!reportVideo) {
130   - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的视频数据失败");
  132 + log.error(format + "的视频数据失败");
131 133 return;
132 134 }
133 135 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
134 136 if (!reportLive) {
135   - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败");
  137 + log.error(format + "的直播数据失败");
136 138 return;
137 139 }
138 140 this.afterCrawl(accountService.findById(account.getId()));
... ... @@ -158,24 +160,24 @@ public class CrawlBizService {
158 160 int hasTryTimes = 0;
159 161 int maxTryTimes = 2;
160 162 while (!result && hasTryTimes < maxTryTimes) {
161   - result = accountRpcService.pushExpireAccount(account.getPhoneNo(), account.getType());
  163 + result = accountRpcService.pushExpireAccount(account.getAccountNo(), account.getType());
162 164 hasTryTimes++;
163 165 }
164 166 }
165 167  
166 168 /**
167   - * 上报粉丝数
  169 + * 上报账户信息
168 170 *
169   - * @param account 账户
170   - * @param fansCnt 粉丝数
  171 + * @param account 账户
  172 + * @param accountMsg 账户其他信息
171 173 * @return
172 174 */
173   - private boolean doReportFansCnt(Account account, Integer fansCnt) {
  175 + private boolean doReportAccountMsg(Account account, ReportAccountDto accountMsg) {
174 176 boolean reportFansCnt;
175 177 int hasTryTimes = 0;
176 178 int maxTryTimes = 2;
177 179 while (hasTryTimes < maxTryTimes) {
178   - reportFansCnt = reportRpcService.reportFansCnt(account.getPhoneNo(), account.getType(), fansCnt);
  180 + reportFansCnt = reportRpcService.reportAccountMsg(account.getAccountNo(), account.getType(), accountMsg);
179 181 if (reportFansCnt) {
180 182 return true;
181 183 } else {
... ... @@ -198,7 +200,7 @@ public class CrawlBizService {
198 200 int hasTryTimes = 0;
199 201 int maxTryTimes = 2;
200 202 while (hasTryTimes < maxTryTimes) {
201   - reportVideo = reportRpcService.reportVideo(account.getPhoneNo(), account.getType(), allVideoMsg);
  203 + reportVideo = reportRpcService.reportVideo(account.getAccountNo(), account.getType(), allVideoMsg);
202 204 if (reportVideo) {
203 205 return true;
204 206 } else {
... ... @@ -221,7 +223,7 @@ public class CrawlBizService {
221 223 int hasTryTimes = 0;
222 224 int maxTryTimes = 2;
223 225 while (hasTryTimes < maxTryTimes) {
224   - reportLive = reportRpcService.reportLive(account.getPhoneNo(), account.getType(), yesterdayLiveMsg);
  226 + reportLive = reportRpcService.reportLive(account.getAccountNo(), account.getType(), yesterdayLiveMsg);
225 227 if (reportLive) {
226 228 return true;
227 229 } else {
... ... @@ -243,14 +245,14 @@ public class CrawlBizService {
243 245 public void reportLive(Account account) throws IOException {
244 246 log.info(Thread.currentThread().getName() + " spring监听器在CrawlBizService.reportLive()方法上成功收到消息: " + account);
245 247 Integer type = account.getType();
246   - final String accountNo = account.getPhoneNo();
  248 + final String accountNo = account.getAccountNo();
247 249 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
248 250 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
249 251 final List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息
250 252 final boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
251 253 if (!reportLive) {
252 254 accountService.setAccountUndone(accountNo);
253   - log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + (account.getType() == 1 ? "快手" : "抖音") + "的直播数据失败");
  255 + log.error(LocalDate.now() + " 上报账户为" + accountNo + "的" + AccountTypeEnum.getNameByValue(account.getType()) + "的直播数据失败");
254 256 }
255 257 }
256 258 }
... ...
src/main/java/cn/fw/freya/service/crawl/CrawlStrategy.java
  View file @d472478
... ... @@ -3,6 +3,7 @@ package cn.fw.freya.service.crawl;
3 3 import cn.fw.freya.enums.AccountTypeEnum;
4 4 import cn.fw.freya.model.data.pool.LivePool;
5 5 import cn.fw.freya.model.data.pool.VideoPool;
  6 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
6 7 import cn.fw.freya.utils.http.InitUserAgent;
7 8  
8 9 import java.io.IOException;
... ... @@ -102,11 +103,11 @@ public interface CrawlStrategy {
102 103 List<LivePool> getYesterdayLiveMsg(String accountNo) throws IOException;
103 104  
104 105 /**
105   - * 更新账户粉丝数
  106 + * 更新账户信息
106 107 *
107 108 * @param accountNo 账户号
108 109 * @return
109 110 * @throws IOException
110 111 */
111   - Integer updateAccountFans(String accountNo) throws IOException;
  112 + ReportAccountDto updateAccountMsg(String accountNo) throws IOException;
112 113 }
... ...
src/main/java/cn/fw/freya/service/crawl/impl/BilibiliCrawl.java
  View file @d472478
... ... @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account;
10 10 import cn.fw.freya.model.data.FwCookie;
11 11 import cn.fw.freya.model.data.pool.LivePool;
12 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 14 import cn.fw.freya.service.crawl.CrawlStrategy;
14 15 import cn.fw.freya.service.data.AccountService;
15 16 import cn.fw.freya.utils.DateUtil;
... ... @@ -154,7 +155,7 @@ public class BilibiliCrawl implements CrawlStrategy {
154 155 }
155 156 } while (currentPage * pageSize < total);// 条件为false, 退出
156 157 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray)));
157   - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue());
  158 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.BILIBILI.getValue(), ResourceTypeEnum.VIDEO.getValue());
158 159 // 视频数据存库
159 160 List<VideoPool> videoPoolList = new ArrayList<>();
160 161 StringBuilder sb = new StringBuilder();
... ... @@ -189,7 +190,7 @@ public class BilibiliCrawl implements CrawlStrategy {
189 190 .playCount(playCnt)// 播放次数
190 191 .likeCount(Optional.ofNullable(stat.getInteger("like")).orElse(0))// 点赞数
191 192 .commentCount(Optional.ofNullable(stat.getInteger("reply")).orElse(0))// 评论数
192   - .phoneNo(accountNo)
  193 + .accountNo(accountNo)
193 194 .reportDate(new Date())
194 195 .publishTime(new Date(Archive.getLong("ctime") * 1000))// 发布时间
195 196 .videoUrl("https://www.bilibili.com/video/" + videoId)// 视频播放地址
... ... @@ -207,7 +208,7 @@ public class BilibiliCrawl implements CrawlStrategy {
207 208 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
208 209 } else {
209 210 final VideoPool nullVideo = VideoPool.builder()
210   - .phoneNo(accountNo)
  211 + .accountNo(accountNo)
211 212 .type(this.getType().getValue())
212 213 .resourceType(ResourceTypeEnum.VIDEO.getValue())
213 214 .reportDate(new Date())
... ... @@ -367,7 +368,7 @@ public class BilibiliCrawl implements CrawlStrategy {
367 368 */
368 369 livePoolList.add(LivePool.builder()
369 370 .type(this.getType().getValue())
370   - .phoneNo(accountNo)
  371 + .accountNo(accountNo)
371 372 .reportDate(new Date())
372 373 .commentUserCnt(Optional.ofNullable(obj.getInteger("danmu_num")).orElse(0))// 评论数
373 374 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
... ... @@ -400,7 +401,7 @@ public class BilibiliCrawl implements CrawlStrategy {
400 401 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的Bilibili直播数据");
401 402 final LivePool nullLive = LivePool.builder()
402 403 .type(this.getType().getValue())
403   - .phoneNo(accountNo)
  404 + .accountNo(accountNo)
404 405 .reportDate(new Date())
405 406 .build();
406 407 livePoolDao.save(nullLive);
... ... @@ -419,10 +420,10 @@ public class BilibiliCrawl implements CrawlStrategy {
419 420 */
420 421 @Override
421 422 @Transactional
422   - public Integer updateAccountFans(String accountNo) throws IOException {
423   - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
424   - if (Objects.nonNull(hasFoundFansCnt)) {
425   - return hasFoundFansCnt;
  423 + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException {
  424 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  425 + if (Objects.nonNull(hasFoundAccountMsg)) {
  426 + return hasFoundAccountMsg;
426 427 }
427 428 HttpConfig config = HttpConfig.custom()
428 429 .url("https://member.bilibili.com/x/web/index/stat")
... ... @@ -442,28 +443,37 @@ public class BilibiliCrawl implements CrawlStrategy {
442 443 if (!StringUtils.hasText(res)) {
443 444 throw new BusinessException("调用Bilibili[粉丝]接口失败");
444 445 }
445   - assert response != null;
446   - Integer fansCnt;
  446 + ReportAccountDto reportAccountDto;
447 447 try {
448   - fansCnt = response.getJSONObject("data").getInteger("total_fans");
  448 + JSONObject data = response.getJSONObject("data");
  449 + reportAccountDto = ReportAccountDto.builder()
  450 + .fansCnt(data.getInteger("total_fans"))
  451 + //.followCnt(data.getInteger("followCnt"))
  452 + .likeCnt(data.getInteger("total_like"))
  453 + //.accountId(data.getString("userId"))
  454 + //.userOtherId(data.getString("userKwaiId"))
  455 + //.accountName(data.getString("userName"))
  456 + .build();
449 457 } catch (Exception e) {
450 458 log.error("获取[哔哩哔哩]fansCnt发生错误", e);
451   - fansCnt = 0;
  459 + reportAccountDto = ReportAccountDto.builder()
  460 + .fansCnt(0)
  461 + .build();
452 462 }
453   - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());
454   - if (Objects.nonNull(account)) {
455   - accountDao.updateFans(account.getId(), fansCnt, new Date());
456   - } else {
  463 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  464 + if (Objects.nonNull(account))
  465 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
  466 + else
457 467 accountDao.save(Account.builder()
458 468 .cookiesStatus(true)
459   - .phoneNo(accountNo)
  469 + .accountNo(accountNo)
460 470 .type(this.getType().getValue())
461   - .fansCnt(fansCnt)
  471 + .fansCnt(reportAccountDto.getFansCnt())
  472 + .accountName(reportAccountDto.getAccountName())
462 473 .reportDate(new Date())
463 474 .done(false)
464 475 .build());
465   - }
466   - return fansCnt;
  476 + return reportAccountDto;
467 477 }
468 478  
469 479 /**
... ... @@ -563,11 +573,11 @@ public class BilibiliCrawl implements CrawlStrategy {
563 573 /**
564 574 * 根据账户号查到用户cookies, 并组装成String形式返回
565 575 *
566   - * @param phoneNo 账户号
  576 + * @param accountNo 账户号
567 577 * @return
568 578 */
569   - public String getUserCookies(String phoneNo) {
570   - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue()));
  579 + public String getUserCookies(String accountNo) {
  580 + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue()));
571 581 }
572 582  
573 583 /**
... ...
src/main/java/cn/fw/freya/service/crawl/impl/Common.java
  View file @d472478
... ... @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.FwCookie;
10 10 import cn.fw.freya.model.data.ResponseReceived;
11 11 import cn.fw.freya.model.data.pool.LivePool;
12 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 14 import com.alibaba.fastjson.JSON;
14 15 import com.alibaba.fastjson.JSONObject;
15 16 import lombok.RequiredArgsConstructor;
... ... @@ -159,7 +160,7 @@ public class Common {
159 160 * @param type 账户类型
160 161 */
161 162 public void deleteCookies(String accountNo, Integer type) {
162   - cookieDao.deleteByPhoneNoAndType(accountNo, type);
  163 + cookieDao.deleteByAccountNoAndType(accountNo, type);
163 164 }
164 165  
165 166 /**
... ... @@ -169,7 +170,7 @@ public class Common {
169 170 * @return
170 171 */
171 172 public List<FwCookie> loadCookie(String accountNo, Integer type) {
172   - return Optional.ofNullable(cookieDao.findByPhoneNoAndType(accountNo, type))
  173 + return Optional.ofNullable(cookieDao.findByAccountNoAndType(accountNo, type))
173 174 .orElse(new ArrayList<>());
174 175 }
175 176  
... ... @@ -252,10 +253,14 @@ public class Common {
252 253 * @param reportDate 指定日期
253 254 * @return
254 255 */
255   - public Integer getHasFoundFansCnt(String accountNo, Integer type, Date reportDate) {
256   - final Integer hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate);
257   - if (Objects.nonNull(hasReportDate)) {
258   - return hasReportDate;
  256 + public ReportAccountDto getHasFoundAccountMsg(String accountNo, Integer type, Date reportDate) {
  257 + final List<Account> hasReportDate = accountDao.getHasReportDate(accountNo, type, reportDate);
  258 + if (!CollectionUtils.isEmpty(hasReportDate)) {
  259 + final Account account = hasReportDate.get(0);
  260 + return ReportAccountDto.builder()
  261 + .fansCnt(account.getFansCnt())
  262 + .accountName(account.getAccountName())
  263 + .build();
259 264 }
260 265 return null;
261 266 }
... ... @@ -269,9 +274,9 @@ public class Common {
269 274 * @return
270 275 */
271 276 public List<VideoPool> getHasFoundVideo(String accountNo, Integer type, Date reportDate) {
272   - final List<VideoPool> hasReportDate = videoPoolDao.getHasReportDate(accountNo, type, reportDate);
273   - if (hasReportDate.size() > 0) {
274   - return hasReportDate;
  277 + final List<VideoPool> hasReportData = videoPoolDao.getHasReportDate(accountNo, type, reportDate);
  278 + if (hasReportData.size() > 0) {
  279 + return hasReportData;
275 280 }
276 281 return null;
277 282 }
... ... @@ -285,9 +290,9 @@ public class Common {
285 290 * @return
286 291 */
287 292 public List<LivePool> getHasFoundLive(String accountNo, Integer type, Date reportDate) {
288   - final List<LivePool> hasReportDate = livePoolDao.getHasReportDate(accountNo, type, reportDate);
289   - if (hasReportDate.size() > 0) {
290   - return hasReportDate;
  293 + final List<LivePool> hasReportData = livePoolDao.getHasReportData(accountNo, type, reportDate);
  294 + if (hasReportData.size() > 0) {
  295 + return hasReportData;
291 296 }
292 297 return null;
293 298 }
... ...
src/main/java/cn/fw/freya/service/crawl/impl/DongCheDiCrawl.java
  View file @d472478
... ... @@ -10,6 +10,7 @@ import cn.fw.freya.model.data.Account;
10 10 import cn.fw.freya.model.data.FwCookie;
11 11 import cn.fw.freya.model.data.pool.LivePool;
12 12 import cn.fw.freya.model.data.pool.VideoPool;
  13 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
13 14 import cn.fw.freya.service.crawl.CrawlStrategy;
14 15 import cn.fw.freya.service.data.AccountService;
15 16 import cn.fw.freya.utils.DateUtil;
... ... @@ -183,7 +184,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
183 184 assert response1 != null;
184 185 JSONArray smallVideoJsonArray = Optional.ofNullable(response1.getJSONArray("item_datas")).orElse(new JSONArray());// 获取[小视频]数据数组
185 186 videoJsonArray.addAll(smallVideoJsonArray);
186   - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue());
  187 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, this.getType().getValue(), ResourceTypeEnum.VIDEO.getValue());
187 188 // 视频数据存库
188 189 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size());
189 190 log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(videoJsonArray)));
... ... @@ -215,7 +216,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
215 216 .playCount(playCount)// 播放次数
216 217 .likeCount(Optional.ofNullable(DCSObj.getInteger("digg_count")).orElse(0))// 点赞数
217 218 .commentCount(Optional.ofNullable(DCSObj.getInteger("comment_count")).orElse(0))// 评论数
218   - .phoneNo(accountNo)
  219 + .accountNo(accountNo)
219 220 .reportDate(new Date())
220 221 .publishTime(new Date(obj.getLong("create_time") * 1000L))// 发布时间
221 222 .videoUrl("https://www.ixigua.com/" + videoId)// 播放地址
... ... @@ -232,7 +233,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
232 233 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
233 234 } else {
234 235 final VideoPool nullVideo = VideoPool.builder()
235   - .phoneNo(accountNo)
  236 + .accountNo(accountNo)
236 237 .type(this.getType().getValue())
237 238 .resourceType(ResourceTypeEnum.VIDEO.getValue())
238 239 .reportDate(new Date())
... ... @@ -285,7 +286,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
285 286 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据");
286 287 final LivePool nullLive = LivePool.builder()
287 288 .type(this.getType().getValue())
288   - .phoneNo(accountNo)
  289 + .accountNo(accountNo)
289 290 .reportDate(new Date())
290 291 .build();
291 292 livePoolDao.save(nullLive);
... ... @@ -326,7 +327,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
326 327 assert userInfo != null;
327 328 livePoolList.add(LivePool.builder()
328 329 .type(this.getType().getValue())
329   - .phoneNo(accountNo)
  330 + .accountNo(accountNo)
330 331 .reportDate(new Date())
331 332 .commentUserCnt(commentCnt)// 评论数
332 333 //.consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
... ... @@ -359,7 +360,7 @@ public class DongCheDiCrawl implements CrawlStrategy {
359 360 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的懂车帝直播数据");
360 361 final LivePool nullLive = LivePool.builder()
361 362 .type(this.getType().getValue())
362   - .phoneNo(accountNo)
  363 + .accountNo(accountNo)
363 364 .reportDate(new Date())
364 365 .build();
365 366 livePoolDao.save(nullLive);
... ... @@ -378,14 +379,19 @@ public class DongCheDiCrawl implements CrawlStrategy {
378 379 * @return
379 380 */
380 381 @Override
381   - public Integer updateAccountFans(String accountNo) {
382   - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
383   - if (Objects.nonNull(hasFoundFansCnt)) {
384   - return hasFoundFansCnt;
  382 + public ReportAccountDto updateAccountMsg(String accountNo) {
  383 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  384 + if (Objects.nonNull(hasFoundAccountMsg)) {
  385 + return hasFoundAccountMsg;
385 386 }
386 387 final String userId;
  388 + String userNick;
  389 + String ttId;
387 390 try {
388   - userId = Objects.requireNonNull(this.getUserInfo(accountNo)).getString("userId");
  391 + final JSONObject obj = Objects.requireNonNull(this.getUserInfo(accountNo));
  392 + userId = obj.getString("userId");
  393 + userNick = obj.getString("userNick");
  394 + ttId = obj.getString("ttId");
389 395 } catch (NullPointerException e) {
390 396 return null;
391 397 }
... ... @@ -414,24 +420,32 @@ public class DongCheDiCrawl implements CrawlStrategy {
414 420 throw new BusinessException("调用懂车帝[粉丝]接口失败");
415 421 }
416 422 final JSONArray data = response1.getJSONArray("data");
  423 + ReportAccountDto reportAccountDto;
417 424 for (Object item : data) {
418 425 JSONObject obj = (JSONObject) item;
419 426 if (obj.getString("title").contains("总数")) {
420   - final Integer fansCnt = obj.getInteger("data");
421   - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());
422   - if (Objects.nonNull(account)) {
423   - accountDao.updateFans(account.getId(), fansCnt, new Date());
424   - } else {
  427 + reportAccountDto = ReportAccountDto.builder()
  428 + .fansCnt(obj.getInteger("data"))
  429 + //.followCnt(obj.getInteger("followCnt"))
  430 + //.likeCnt(obj.getInteger("likeCnt"))
  431 + .accountId(userId)
  432 + .userOtherId(ttId)
  433 + .accountName(userNick)
  434 + .build();
  435 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  436 + if (Objects.nonNull(account))
  437 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
  438 + else
425 439 accountDao.save(Account.builder()
426 440 .cookiesStatus(true)
427   - .phoneNo(accountNo)
  441 + .accountNo(accountNo)
428 442 .type(this.getType().getValue())
429   - .fansCnt(fansCnt)
  443 + .fansCnt(reportAccountDto.getFansCnt())
  444 + .accountName(reportAccountDto.getAccountName())
430 445 .reportDate(new Date())
431 446 .done(false)
432 447 .build());
433   - }
434   - return fansCnt;
  448 + return reportAccountDto;
435 449 }
436 450 }
437 451 return null;
... ... @@ -559,11 +573,11 @@ public class DongCheDiCrawl implements CrawlStrategy {
559 573 /**
560 574 * 根据账户号查到用户cookies, 并组装成String形式返回
561 575 *
562   - * @param phoneNo 账户号
  576 + * @param accountNo 账户号
563 577 * @return
564 578 */
565   - public String getUserCookies(String phoneNo) {
566   - return this.processCookiesToString(common.loadCookie(phoneNo, this.getType().getValue()));
  579 + public String getUserCookies(String accountNo) {
  580 + return this.processCookiesToString(common.loadCookie(accountNo, this.getType().getValue()));
567 581 }
568 582  
569 583 /**
... ...
src/main/java/cn/fw/freya/service/crawl/impl/DouYinCrawl.java
  View file @d472478
... ... @@ -13,6 +13,7 @@ import cn.fw.freya.model.data.LiveOverview;
13 13 import cn.fw.freya.model.data.ResponseReceived;
14 14 import cn.fw.freya.model.data.pool.LivePool;
15 15 import cn.fw.freya.model.data.pool.VideoPool;
  16 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
16 17 import cn.fw.freya.service.crawl.CrawlStrategy;
17 18 import cn.fw.freya.service.data.AccountService;
18 19 import cn.fw.freya.utils.DateUtil;
... ... @@ -132,13 +133,13 @@ public class DouYinCrawl implements CrawlStrategy {
132 133 */
133 134 @Override
134 135 @Transactional
135   - /*public List<VideoPool> getAllVideoMsg(String phoneNo) {
136   - final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(phoneNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  136 + /*public List<VideoPool> getAllVideoMsg(String accountNo) {
  137 + final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
137 138 if (Objects.nonNull(hasFoundVideo)) {
138 139 return hasFoundVideo;
139 140 }
140 141 final String uuid = UUID.randomUUID().toString().replace("-", "");
141   - final WebDriver driver = this.getDYDriver(phoneNo, uuid);
  142 + final WebDriver driver = this.getDYDriver(accountNo, uuid);
142 143 String targetUrl = "https://creator.douyin.com/creator-micro/content/manage";
143 144 driver.get(targetUrl);
144 145 List<WebElement> endNodes;
... ... @@ -146,7 +147,7 @@ public class DouYinCrawl implements CrawlStrategy {
146 147 do {
147 148 endNodes = driver.findElements(By.xpath("//div[starts-with(@class,'load-more')]/div[text()='没有更多视频']"));
148 149 if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
149   - this.exitBrowser(phoneNo, uuid);
  150 + this.exitBrowser(accountNo, uuid);
150 151 return null;
151 152 }
152 153 if (CollectionUtils.isEmpty(endNodes)) {
... ... @@ -211,7 +212,7 @@ public class DouYinCrawl implements CrawlStrategy {
211 212 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
212 213 .duration(jsonObj.getLong("duration") / (1000 * 1.0))
213 214 .likeCount(statistics.getInteger("share_count"))
214   - .phoneNo(phoneNo)
  215 + .accountNo(accountNo)
215 216 .playCount(statistics.getInteger("play_count"))
216 217 .reportDate(new Date())
217 218 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
... ... @@ -234,7 +235,7 @@ public class DouYinCrawl implements CrawlStrategy {
234 235 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
235 236 } else {
236 237 videoPoolDao.save(VideoPool.builder()
237   - .phoneNo(phoneNo)
  238 + .accountNo(accountNo)
238 239 .type(this.getType().getValue())
239 240 .resourceType(ResourceTypeEnum.VIDEO.getValue())
240 241 .reportDate(new Date())
... ... @@ -243,9 +244,9 @@ public class DouYinCrawl implements CrawlStrategy {
243 244 log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息");
244 245 } catch (Exception e) {
245 246 assert log != null;
246   - log.error(LocalDate.now() + " 执行保存" + phoneNo + "的抖音视频数据失败", e);
  247 + log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败", e);
247 248 }
248   - this.exitBrowser(phoneNo, uuid);
  249 + this.exitBrowser(accountNo, uuid);
249 250 return videoPoolList;
250 251 }*/
251 252 public List<VideoPool> getAllVideoMsg(String accountNo) {
... ... @@ -279,7 +280,7 @@ public class DouYinCrawl implements CrawlStrategy {
279 280 if (element.getText().contains("没有") || element.getText().contains("起开始展示数据")) {
280 281 this.exitBrowser(accountNo, uuid);
281 282 videoPoolDao.save(VideoPool.builder()
282   - .phoneNo(accountNo)
  283 + .accountNo(accountNo)
283 284 .type(this.getType().getValue())
284 285 .resourceType(ResourceTypeEnum.VIDEO.getValue())
285 286 .reportDate(new Date())
... ... @@ -358,7 +359,7 @@ public class DouYinCrawl implements CrawlStrategy {
358 359 .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
359 360 .duration(Optional.ofNullable(jsonObj.getDouble("duration")).orElse(0d) / 1000)
360 361 .likeCount(Optional.ofNullable(statistics.getInteger("digg_count")).orElse(0))
361   - .phoneNo(accountNo)
  362 + .accountNo(accountNo)
362 363 .playCount(Optional.ofNullable(statistics.getInteger("play_count")).orElse(0))
363 364 .reportDate(new Date())
364 365 .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
... ... @@ -390,7 +391,7 @@ public class DouYinCrawl implements CrawlStrategy {
390 391 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
391 392 } else {
392 393 videoPoolDao.save(VideoPool.builder()
393   - .phoneNo(accountNo)
  394 + .accountNo(accountNo)
394 395 .type(this.getType().getValue())
395 396 .resourceType(ResourceTypeEnum.VIDEO.getValue())
396 397 .reportDate(new Date())
... ... @@ -449,7 +450,7 @@ public class DouYinCrawl implements CrawlStrategy {
449 450 .type(this.getType().getValue())
450 451 .reportDate(new Date())
451 452 .uploadTime(new Date())
452   - .phoneNo(accountNo)
  453 + .accountNo(accountNo)
453 454 .liveCnt(summarizeData.getInteger("yes_live_cnt"))
454 455 .liveDuration(summarizeData.getDouble("yes_live_duration"))
455 456 .watchTimesCnt(summarizeData.getInteger("yes_watch_cnt"))
... ... @@ -514,7 +515,7 @@ public class DouYinCrawl implements CrawlStrategy {
514 515 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的抖音直播数据");
515 516 final LivePool nullLive = LivePool.builder()
516 517 .type(this.getType().getValue())
517   - .phoneNo(accountNo)
  518 + .accountNo(accountNo)
518 519 .reportDate(new Date())
519 520 .build();
520 521 livePoolDao.save(nullLive);
... ... @@ -567,7 +568,7 @@ public class DouYinCrawl implements CrawlStrategy {
567 568 */
568 569 livePoolList.add(LivePool.builder()
569 570 .type(this.getType().getValue())
570   - .phoneNo(accountNo)
  571 + .accountNo(accountNo)
571 572 .reportDate(new Date())
572 573 .commentUserCnt(Optional.ofNullable(obj.getInteger("comment_ucnt")).orElse(0))// 评论人数
573 574 .consumeUserCnt(Optional.ofNullable(obj.getInteger("consume_ucnt")).orElse(0))// 付费人数
... ... @@ -613,7 +614,7 @@ public class DouYinCrawl implements CrawlStrategy {
613 614 } else {
614 615 livePoolDao.save(LivePool.builder()
615 616 .type(this.getType().getValue())
616   - .phoneNo(accountNo)
  617 + .accountNo(accountNo)
617 618 .reportDate(new Date())
618 619 .build());
619 620 }
... ... @@ -633,10 +634,10 @@ public class DouYinCrawl implements CrawlStrategy {
633 634 */
634 635 @Override
635 636 @Transactional
636   - public Integer updateAccountFans(String accountNo) {
637   - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
638   - if (Objects.nonNull(hasFoundFansCnt)) {
639   - return hasFoundFansCnt;
  637 + public ReportAccountDto updateAccountMsg(String accountNo) {
  638 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  639 + if (Objects.nonNull(hasFoundAccountMsg)) {
  640 + return hasFoundAccountMsg;
640 641 }
641 642 final String uuid = UUID.randomUUID().toString().replace("-", "");
642 643 final WebDriver driver = this.getDYDriver(accountNo, uuid);
... ... @@ -678,9 +679,9 @@ public class DouYinCrawl implements CrawlStrategy {
678 679 // 读http日志, 处理数据
679 680 final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
680 681 String dataUrl = "https://creator.douyin.com/web/api/media/user/info";// 数据接口地址
681   - final Integer[] fansCnt = new Integer[1];
  682 + AtomicReference<ReportAccountDto> reportAccountDto = new AtomicReference<>();
682 683 responseReceivedEvents.forEach(item -> {
683   - JSONObject userInfo;
  684 + JSONObject data;
684 685 try {
685 686 final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
686 687 if (Objects.nonNull(response)) {
... ... @@ -690,26 +691,32 @@ public class DouYinCrawl implements CrawlStrategy {
690 691 if (Objects.isNull(body)) {
691 692 return;
692 693 }
693   - userInfo = body.getJSONObject("user");
694   - if (Objects.isNull(userInfo)) {
  694 + data = body.getJSONObject("user");
  695 + if (Objects.isNull(data)) {
695 696 return;
696 697 }
697   - fansCnt[0] = userInfo.getInteger("follower_count");
698   - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());
699   - if (Objects.nonNull(fansCnt[0])) {
700   - if (Objects.nonNull(account)) {
701   - accountDao.updateFans(account.getId(), fansCnt[0], new Date());
702   - } else {
703   - accountDao.save(Account.builder()
704   - .cookiesStatus(true)
705   - .phoneNo(accountNo)
706   - .type(this.getType().getValue())
707   - .fansCnt(fansCnt[0])
708   - .reportDate(new Date())
709   - .done(false)
710   - .build());
711   - }
712   - }
  698 + reportAccountDto.set(ReportAccountDto.builder()
  699 + .fansCnt(data.getInteger("follower_count"))
  700 + .followCnt(data.getInteger("following_count"))
  701 + .likeCnt(data.getInteger("total_favorited"))
  702 + .accountId(data.getString("unique_id"))
  703 + .userOtherId(data.getString("uid"))
  704 + .accountName(data.getString("nickname"))
  705 + .build()
  706 + );
  707 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
  708 + if (Objects.nonNull(account))
  709 + accountDao.updateMsg(account.getId(), reportAccountDto.get().getAccountName(), reportAccountDto.get().getFansCnt(), new Date());
  710 + else
  711 + accountDao.save(Account.builder()
  712 + .cookiesStatus(true)
  713 + .accountNo(accountNo)
  714 + .type(this.getType().getValue())
  715 + .fansCnt(reportAccountDto.get().getFansCnt())
  716 + .accountName(reportAccountDto.get().getAccountName())
  717 + .reportDate(new Date())
  718 + .done(false)
  719 + .build());
713 720 }
714 721 } catch (Exception e) {
715 722 this.exitBrowser(accountNo, uuid);
... ... @@ -718,7 +725,7 @@ public class DouYinCrawl implements CrawlStrategy {
718 725 }
719 726 });
720 727 this.exitBrowser(accountNo, uuid);
721   - return fansCnt[0];
  728 + return reportAccountDto.get();
722 729 }
723 730  
724 731 /**
... ...
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @d472478
... ... @@ -12,6 +12,7 @@ import cn.fw.freya.model.data.FwCookie;
12 12 import cn.fw.freya.model.data.ResponseReceived;
13 13 import cn.fw.freya.model.data.pool.LivePool;
14 14 import cn.fw.freya.model.data.pool.VideoPool;
  15 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
15 16 import cn.fw.freya.service.crawl.CrawlStrategy;
16 17 import cn.fw.freya.service.data.AccountService;
17 18 import cn.fw.freya.utils.DateUtil;
... ... @@ -30,20 +31,29 @@ import lombok.RequiredArgsConstructor;
30 31 import lombok.extern.slf4j.Slf4j;
31 32 import org.apache.http.client.CookieStore;
32 33 import org.apache.http.impl.client.BasicCookieStore;
  34 +import org.apache.logging.log4j.util.PropertiesUtil;
33 35 import org.openqa.selenium.By;
34 36 import org.openqa.selenium.Cookie;
35 37 import org.openqa.selenium.WebDriver;
36 38 import org.openqa.selenium.WebElement;
37 39 import org.openqa.selenium.support.ui.WebDriverWait;
  40 +import org.springframework.context.SmartLifecycle;
  41 +import org.springframework.core.io.ClassPathResource;
  42 +import org.springframework.core.io.support.PropertiesLoaderUtils;
38 43 import org.springframework.stereotype.Service;
39 44 import org.springframework.transaction.annotation.Transactional;
40 45 import org.springframework.util.CollectionUtils;
41 46 import org.springframework.util.StringUtils;
42 47  
43 48 import javax.annotation.Resource;
  49 +import java.io.BufferedWriter;
  50 +import java.io.FileOutputStream;
44 51 import java.io.IOException;
  52 +import java.io.OutputStreamWriter;
45 53 import java.math.BigDecimal;
46 54 import java.math.RoundingMode;
  55 +import java.net.URLDecoder;
  56 +import java.nio.charset.StandardCharsets;
47 57 import java.time.LocalDate;
48 58 import java.time.LocalDateTime;
49 59 import java.util.*;
... ... @@ -62,8 +72,9 @@ import java.util.stream.Collectors;
62 72 @Service
63 73 @RequiredArgsConstructor
64 74 @SuppressWarnings("Duplicates")
65   -public class KuaiShouCrawl implements CrawlStrategy {
  75 +public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
66 76  
  77 + private boolean isRunning = false;
67 78 private final VideoPoolDao videoPoolDao;
68 79 private final LivePoolDao livePoolDao;
69 80 private final AccountDao accountDao;
... ... @@ -89,25 +100,46 @@ public class KuaiShouCrawl implements CrawlStrategy {
89 100 final List<Account> accountList = accountDao.getAllKSAccount();
90 101 accountList.forEach(item ->
91 102 Arrays.stream(DataTypeEnum.values()).forEach(item1 ->
92   - threadPoolExecutor.execute(() -> {
93   - final String accountNo = item.getPhoneNo();
94   - final Integer typeValue = item1.getValue();
95   - String key = accountNo + "#" + typeValue;
96   - final String ns_sig3 = this.getNS_sig3(accountNo, typeValue);
97   - if (Objects.nonNull(ns_sig3))
98   - sig3Map.put(key, ns_sig3);
99   - })
  103 + threadPoolExecutor.execute(() -> this.task(item.getAccountNo(), item1.getValue()))
100 104 )
101 105 );
102 106 return true;
103 107 }
104 108  
  109 + /**
  110 + * 获取类型签名task
  111 + *
  112 + * @param accountNo 账户号
  113 + * @param dataType 数据类型
  114 + */
  115 + public void task(String accountNo, Integer dataType) {
  116 + String key = accountNo + "#" + dataType;
  117 + final String ns_sig3 = this.getNS_sig3(accountNo, dataType, true);
  118 + if (Objects.nonNull(ns_sig3))
  119 + sig3Map.put(key, ns_sig3);
  120 + }
  121 +
  122 + /**
  123 + * 停止设置sig3Map线程池
  124 + *
  125 + * @return
  126 + */
105 127 public boolean stopSetSig3Map() {
106 128 threadPoolExecutor.shutdownNow();
107 129 return true;
108 130 }
109 131  
110 132 /**
  133 + * 清空sig3Map
  134 + *
  135 + * @return
  136 + */
  137 + public boolean cleanSig3Map() {
  138 + sig3Map.clear();
  139 + return true;
  140 + }
  141 +
  142 + /**
111 143 * 获取sig3Map
112 144 */
113 145 public String getSig3Map() {
... ... @@ -117,8 +149,13 @@ public class KuaiShouCrawl implements CrawlStrategy {
117 149 /**
118 150 * 设置sig3Map
119 151 */
120   - public boolean setMapFromString(String jsonStr) {
121   - HashMap<String, String> map = JSON.parseObject(JSON.parseObject(jsonStr).getString("jsonStr"), new TypeReference<>() {
  152 + public boolean setMapFromString(String jsonStr, boolean interior) {
  153 + String jsonString;
  154 + if (interior)
  155 + jsonString = jsonStr;
  156 + else
  157 + jsonString = JSON.parseObject(jsonStr).getString("jsonStr");
  158 + HashMap<String, String> map = JSON.parseObject(jsonString, new TypeReference<>() {
122 159 }, Feature.OrderedField);
123 160 map.forEach(sig3Map::put);
124 161 return true;
... ... @@ -215,7 +252,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
215 252 CookieStore cookieStore = new BasicCookieStore();
216 253 cookies.setCookieStore(cookieStore);
217 254 Date previousDay = DateUtil.getPreviousDay(new Date());
218   - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue());
  255 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue(), false);
219 256 if (Objects.isNull(ns_sig3))
220 257 return null;
221 258 Map<String, Object> params = new LinkedHashMap<>();
... ... @@ -243,8 +280,12 @@ public class KuaiShouCrawl implements CrawlStrategy {
243 280 if (!StringUtils.hasText(res)) {
244 281 throw new BusinessException("调用快手[视频]接口失败");
245 282 }
  283 + if (Objects.equals(response.getInteger("result"), 500002)) {
  284 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.VIDEO.getValue()));
  285 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  286 + }
246 287 JSONArray videoJsonArray = Optional.ofNullable(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject()).getJSONArray("photoList")).orElse(new JSONArray());
247   - videoPoolDao.deleteByPhoneNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue());
  288 + videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue());
248 289 // 视频数据存库
249 290 List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size());
250 291 videoJsonArray.forEach(item -> {
... ... @@ -286,7 +327,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
286 327 .playCount(Optional.ofNullable(obj.getInteger("playCount")).orElse(0))//
287 328 .likeCount(Optional.ofNullable(obj.getInteger("likeCount")).orElse(0))//
288 329 .commentCount(Optional.ofNullable(obj.getInteger("commentCount")).orElse(0))//
289   - .phoneNo(accountNo)
  330 + .accountNo(accountNo)
290 331 .reportDate(new Date())
291 332 .publishTime(PublicUtil.parseDate(obj.getString("publishTime")))//
292 333 .videoUrl("https://www.kuaishou.com/short-video/" + obj.getString("workId"))
... ... @@ -305,7 +346,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
305 346 videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
306 347 } else {
307 348 final VideoPool nullVideo = VideoPool.builder()
308   - .phoneNo(accountNo)
  349 + .accountNo(accountNo)
309 350 .type(this.getType().getValue())
310 351 .resourceType(ResourceTypeEnum.VIDEO.getValue())
311 352 .reportDate(new Date())
... ... @@ -341,7 +382,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
341 382 Date endTime = DateUtil.getThisDayMaxTime(previousDay);
342 383 Date startTime = DateUtil.getThisDayMinTime(previousDay);
343 384 //Date startTime = DateUtil.getThisDayMinTime(new Date(previousDay.getTime() - 7 * 24 * 3600 * 1000L));// 补数据使用
344   - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue());
  385 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue(), false);
345 386 if (Objects.isNull(ns_sig3))
346 387 return null;
347 388 Map<String, Object> params = new LinkedHashMap<>();
... ... @@ -371,6 +412,10 @@ public class KuaiShouCrawl implements CrawlStrategy {
371 412 if (!StringUtils.hasText(res)) {
372 413 throw new BusinessException("调用快手[视频]接口失败");
373 414 }
  415 + if (Objects.equals(response.getInteger("result"), 500002)) {
  416 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.LIVE.getValue()));
  417 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  418 + }
374 419 JSONObject dataJSONObject = response.getJSONObject("data");
375 420 JSONArray dataJSONArray = dataJSONObject.getJSONArray("details");
376 421 /*List<JSONObject> collect = new ArrayList<>();
... ... @@ -424,49 +469,9 @@ public class KuaiShouCrawl implements CrawlStrategy {
424 469 * userHead: "https://tx2.a.kwimgs.com/uhead/AB/2021/11/11/12/BMjAyMTExMTExMjUwMzZfMjU2MTc2NDMyMV8xX2hkMjg5XzMyOA==_s.jpg"
425 470 * -userName: "长安汽车。小明聊聊车"
426 471 */
427   - /*String coverUrl = obj.getString("liveCover");// 直播封面地址
428   - String playbackUrl = null;
429   - final Double liveDuration = obj.getDouble("liveDuration");// 数据接口返回直播时长
430   - final Long liveStartTimeStamp = obj.getLong("liveTime");// 数据接口返回开播时间戳
431   - if (liveDuration >= 60) {// 直播时长大于等于60分钟才记录回放信息
432   - if (!CollectionUtils.isEmpty(finalCollect)) {
433   - JSONObject playbackMsg;
434   - if (Objects.equals(finalCollect.size(), 1) && Objects.equals(dataJSONArray.size(), 1)) {
435   - playbackMsg = finalCollect.get(0);
436   - Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长
437   - final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳
438   - final double timeSubAbs = Math.abs(liveDuration - duration);
439   - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
440   - // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等
441   - // (liveStartSub < 2)->说明开播时间几乎一样
442   - if (!((Objects.equals(liveDuration, duration) || (timeSubAbs < 5)) && liveStartSub < 10)) {
443   - playbackMsg = null;
444   - log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
445   - }
446   - } else {
447   - List<JSONObject> collect1 = finalCollect.stream().filter(item1 -> {
448   - Double duration = item1.getDouble("duration");
449   - final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳
450   - final double timeSubAbs = Math.abs(liveDuration - duration);
451   - final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
452   - return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10;
453   - }).collect(Collectors.toList());
454   - if (Objects.equals(collect1.size(), 1)) {
455   - playbackMsg = collect1.get(0);
456   - } else {
457   - playbackMsg = null;
458   - log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
459   - }
460   - }
461   - if (Objects.nonNull(playbackMsg)) {
462   - coverUrl = playbackMsg.getString("coverUrl");
463   - playbackUrl = this.playbackBaseUrl + playbackMsg.getString("productId");
464   - }
465   - }
466   - }*/
467 472 livePoolList.add(LivePool.builder()
468 473 .type(this.getType().getValue())
469   - .phoneNo(accountNo)
  474 + .accountNo(accountNo)
470 475 .reportDate(new Date())
471 476 .commentUserCnt(Optional.ofNullable(obj.getInteger("commentUv")).orElse(0))
472 477 .consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
... ... @@ -499,7 +504,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
499 504 log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的快手直播数据");
500 505 final LivePool nullLive = LivePool.builder()
501 506 .type(this.getType().getValue())
502   - .phoneNo(accountNo)
  507 + .accountNo(accountNo)
503 508 .reportDate(new Date())
504 509 .build();
505 510 livePoolDao.save(nullLive);
... ... @@ -517,7 +522,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
517 522 object.put("playbackUrl", null);
518 523 if (CollectionUtils.isEmpty(collect))
519 524 return object;
520   - final String accountNo = dbLive.getPhoneNo();
  525 + final String accountNo = dbLive.getAccountNo();
521 526 final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长
522 527 final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳
523 528 if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息
... ... @@ -565,8 +570,8 @@ public class KuaiShouCrawl implements CrawlStrategy {
565 570 JSONArray objects = new JSONArray();
566 571 if (Objects.isNull(searchKey))
567 572 return objects;
568   - Map<String, Object> params = new HashMap<>();
569   - Map<String, Object> params1 = new HashMap<>();
  573 + Map<String, Object> params = new LinkedHashMap<>();
  574 + Map<String, Object> params1 = new LinkedHashMap<>();
570 575 params1.put("principalId", searchKey);
571 576 params1.put("pcursor", "");
572 577 params1.put("count", 150);
... ... @@ -610,17 +615,17 @@ public class KuaiShouCrawl implements CrawlStrategy {
610 615 */
611 616 @Override
612 617 @Transactional
613   - public Integer updateAccountFans(String accountNo) throws IOException {
614   - final Integer hasFoundFansCnt = common.getHasFoundFansCnt(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
615   - if (Objects.nonNull(hasFoundFansCnt)) {
616   - //return hasFoundFansCnt;
  618 + public ReportAccountDto updateAccountMsg(String accountNo) throws IOException {
  619 + final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
  620 + if (Objects.nonNull(hasFoundAccountMsg)) {
  621 + return hasFoundAccountMsg;
617 622 }
618 623 HttpCookies cookies = HttpCookies.custom();
619 624 CookieStore cookieStore = new BasicCookieStore();
620 625 cookies.setCookieStore(cookieStore);
621 626 Map<String, Object> params = new HashMap<>();
622 627 params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo));
623   - final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue());
  628 + final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue(), false);
624 629 if (Objects.isNull(ns_sig3))
625 630 return null;
626 631 HttpConfig config = HttpConfig.custom()
... ... @@ -643,24 +648,33 @@ public class KuaiShouCrawl implements CrawlStrategy {
643 648 if (!StringUtils.hasText(res)) {
644 649 throw new BusinessException("调用快手[直播]接口失败");
645 650 }
646   - Integer fansNum;
647   - if (Objects.equals(response.getInteger("result"), 500002))
648   - fansNum = 0;
649   - else
650   - fansNum = response.getJSONObject("data").getInteger("fansCnt");
651   - final Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());
  651 + if (Objects.equals(response.getInteger("result"), 500002)) {
  652 + threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.FANS.getValue()));
  653 + throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
  654 + }
  655 + JSONObject data = response.getJSONObject("data");
  656 + ReportAccountDto reportAccountDto = ReportAccountDto.builder()
  657 + .fansCnt(data.getInteger("fansCnt"))
  658 + .followCnt(data.getInteger("followCnt"))
  659 + .likeCnt(data.getInteger("likeCnt"))
  660 + .accountId(data.getString("userId"))
  661 + .userOtherId(data.getString("userKwaiId"))
  662 + .accountName(data.getString("userName"))
  663 + .build();
  664 + final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
652 665 if (Objects.nonNull(account))
653   - accountDao.updateFans(account.getId(), fansNum, new Date());
  666 + accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
654 667 else
655 668 accountDao.save(Account.builder()
656 669 .cookiesStatus(true)
657   - .phoneNo(accountNo)
  670 + .accountNo(accountNo)
658 671 .type(this.getType().getValue())
659   - .fansCnt(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt"))
  672 + .fansCnt(reportAccountDto.getFansCnt())
  673 + .accountName(reportAccountDto.getAccountName())
660 674 .reportDate(new Date())
661 675 .done(false)
662 676 .build());
663   - return Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject().fluentPut("fansCnt", 0)).getInteger("fansCnt");
  677 + return reportAccountDto;
664 678 }
665 679  
666 680 /**
... ... @@ -700,12 +714,16 @@ public class KuaiShouCrawl implements CrawlStrategy {
700 714 *
701 715 * @param accountNo 快手账户号
702 716 * @param type 密钥类型(1:粉丝, 2:短视频, 3:直播)
  717 + * @param retryGet 是否重新获取
703 718 */
704   - public String getNS_sig3(String accountNo, Integer type) {
  719 + public String getNS_sig3(String accountNo, Integer type, boolean retryGet) {
705 720 final String key = accountNo + "#" + type;
706   - String NS_sig3 = sig3Map.get(key);
707   - if (StringUtils.hasText(NS_sig3))
708   - return NS_sig3;
  721 + String NS_sig3;
  722 + if (!retryGet) {
  723 + NS_sig3 = sig3Map.get(key);
  724 + if (StringUtils.hasText(NS_sig3))
  725 + return NS_sig3;
  726 + }
709 727 final String uuid = UUID.randomUUID().toString().replace("-", "");
710 728 final WebDriver driver = this.getKSDriver(accountNo, uuid);
711 729 String targetUrl = null;
... ... @@ -852,7 +870,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
852 870 * @return
853 871 */
854 872 public String getRandomUserCookies() {
855   - final List<FwCookie> cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getPhoneNo(), this.getType().getValue());
  873 + final List<FwCookie> cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getAccountNo(), this.getType().getValue());
856 874 StringBuffer sb = new StringBuffer();
857 875 sb.append("clientid=3;");
858 876 cookies.forEach(item -> {
... ... @@ -903,4 +921,78 @@ public class KuaiShouCrawl implements CrawlStrategy {
903 921 return true;
904 922 }
905 923  
  924 + @Override
  925 + public int getPhase() {
  926 + return 1;// 默认为0
  927 + }
  928 +
  929 + @Override
  930 + public boolean isAutoStartup() {
  931 + return true;// 默认为false
  932 + }
  933 +
  934 + @Override
  935 + public void stop(Runnable callback) {
  936 + log.info("springIOC停止, 将签名信息从Map中导出到配置文件");
  937 + callback.run();
  938 + isRunning = false;
  939 + final HashMap<String, String> props = new HashMap<>();
  940 + props.put("KSAccountNS_sig", JSON.toJSONString(sig3Map));
  941 + updateProperties("NS_sig3Msg.properties", props);
  942 + }
  943 +
  944 + @Override
  945 + public void start() {
  946 + log.info("springIOC启动, 将签名信息从配置文件读出并导入到Map中");
  947 + isRunning = true;
  948 + Properties props;
  949 + try {
  950 + props = PropertiesLoaderUtils.loadAllProperties("NS_sig3Msg.properties");
  951 + for (Object key : props.keySet()) {
  952 + if (Objects.equals(key, "KSAccountNS_sig"))
  953 + this.setMapFromString(props.get(key).toString(), true);
  954 + }
  955 + } catch (IOException e) {
  956 + log.error(e.getMessage());
  957 + }
  958 + }
  959 +
  960 + @Override
  961 + public void stop() {
  962 + log.info("stop");
  963 + isRunning = false;
  964 + }
  965 +
  966 + @Override
  967 + public boolean isRunning() {
  968 + return isRunning;
  969 + }
  970 +
  971 + public void updateProperties(String fileName, Map<String, String> keyValueMap) {
  972 + String filePath = Objects.requireNonNull(PropertiesUtil.class.getClassLoader().getResource(fileName)).getFile();
  973 + Properties props;
  974 + BufferedWriter bw = null;
  975 + try {
  976 + filePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8);
  977 + log.debug("updateProperties propertiesPath:" + filePath);
  978 + props = PropertiesLoaderUtils.loadProperties(new ClassPathResource(fileName));
  979 + log.debug("updateProperties old:" + props);
  980 + bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath)));// 写入属性文件
  981 + //props.clear();// 清空旧的文件
  982 + for (String key : keyValueMap.keySet()) {
  983 + props.setProperty(key, keyValueMap.get(key));
  984 + }
  985 + log.debug("updateProperties new:" + props);
  986 + props.store(bw, "");
  987 + } catch (IOException e) {
  988 + log.error(e.getMessage());
  989 + } finally {
  990 + try {
  991 + assert bw != null;
  992 + bw.close();
  993 + } catch (IOException e) {
  994 + e.printStackTrace();
  995 + }
  996 + }
  997 + }
906 998 }
... ...
src/main/java/cn/fw/freya/service/rpc/AccountRpcService.java
  View file @d472478
... ... @@ -35,13 +35,13 @@ public class AccountRpcService {
35 35 /**
36 36 * 账号cookie失效, 上报服务器
37 37 *
38   - * @param phoneNo 账号
39   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  38 + * @param accountNo 账号
  39 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
40 40 * @return 结果
41 41 */
42   - public boolean pushExpireAccount(String phoneNo, Integer type) {
  42 + public boolean pushExpireAccount(String accountNo, Integer type) {
43 43 Map<String, Object> params = new HashMap<>();
44   - params.put("account", phoneNo);
  44 + params.put("account", accountNo);
45 45 params.put("type", type);
46 46 HttpConfig config = HttpConfig.custom()
47 47 .url(getBaseUrl() + PathConstant.EXPIRE_ACCOUNT)
... ... @@ -55,7 +55,7 @@ public class AccountRpcService {
55 55 String message = Optional.ofNullable(resObj.getString("message")).orElse("账号cookie失效上报失败");
56 56 throw new BusinessException(status, message);
57 57 }
58   - accountService.updateAccountCookiesStatus(phoneNo, type, false);
  58 + accountService.updateAccountCookiesStatus(accountNo, type, false);
59 59 return true;
60 60 }
61 61  
... ... @@ -86,7 +86,7 @@ public class AccountRpcService {
86 86 jsonArray.forEach(item -> {
87 87 final JSONObject obj = (JSONObject) item;
88 88 final AccountDto dto = new AccountDto();
89   - dto.setPhoneNo(obj.getString("account"));
  89 + dto.setAccountNo(obj.getString("account"));
90 90 dto.setPlaybackSearchKey(obj.getString("playbackSearchKey"));
91 91 dto.setGroupId(obj.getLong("groupId"));
92 92 list.add(dto);
... ...
src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java
  View file @d472478
... ... @@ -4,6 +4,7 @@ import cn.fw.freya.common.PathConstant;
4 4 import cn.fw.freya.model.data.pool.LivePool;
5 5 import cn.fw.freya.model.data.pool.VideoPool;
6 6 import cn.fw.freya.model.dto.rpc.LivePoolDto;
  7 +import cn.fw.freya.model.dto.rpc.ReportAccountDto;
7 8 import cn.fw.freya.model.dto.rpc.VideoPoolDto;
8 9 import cn.fw.freya.utils.RequestUtil;
9 10 import cn.fw.freya.utils.http.HttpConfig;
... ... @@ -32,18 +33,19 @@ public class ReportRpcService {
32 33 private String baseUrl;
33 34  
34 35 /**
35   - * 账号粉丝数上报
  36 + * 账户信息上报
36 37 *
37   - * @param accountNo 账户号
38   - * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
39   - * @param fans 粉丝数
  38 + * @param accountNo 账户号
  39 + * @param type 账户类型(1:快手, 2:抖音, 3:懂车帝, 4:Bilibili)
  40 + * @param accountMsg 账户信息
40 41 * @return
41 42 */
42   - public boolean reportFansCnt(String accountNo, Integer type, Integer fans) {
  43 + public boolean reportAccountMsg(String accountNo, Integer type, ReportAccountDto accountMsg) {
43 44 Map<String, Object> params = new HashMap<>();
44 45 params.put("account", accountNo);
45 46 params.put("type", type);
46   - params.put("fans", fans);
  47 + params.put("fans", accountMsg.getFansCnt());
  48 + params.put("accountName", accountMsg.getAccountName());
47 49 HttpConfig config = HttpConfig.custom()
48 50 .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName())
49 51 .url(getBaseUrl() + PathConstant.REPORT_FANS)
... ...
src/main/java/cn/fw/freya/task/DataCaptureTask.java
  View file @d472478
... ... @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject;
16 16 import lombok.RequiredArgsConstructor;
17 17 import lombok.extern.slf4j.Slf4j;
18 18 import org.springframework.context.ApplicationEventPublisher;
19   -import org.springframework.scheduling.annotation.Scheduled;
20 19 import org.springframework.stereotype.Component;
21 20 import org.springframework.util.CollectionUtils;
22 21  
... ... @@ -51,7 +50,7 @@ public class DataCaptureTask {
51 50 /**
52 51 * 每分钟执行多线程同时抓取数据
53 52 */
54   - @Scheduled(cron = "0 0/1 * * * ?")
  53 + //@Scheduled(cron = "0 0/1 * * * ?")
55 54 public void capture() {
56 55 final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool();
57 56 BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列
... ... @@ -78,11 +77,11 @@ public class DataCaptureTask {
78 77 });
79 78 final Integer accountType = currentAccount.getType();// 当前账户类型
80 79 final Integer nextAccountType = Optional.ofNullable(nextAccount.getType()).orElse(-1);// 下一个账户类型
81   - final String nextAccountNo = Optional.ofNullable(nextAccount.getPhoneNo()).orElse("");// 下一个账户号
  80 + final String nextAccountNo = Optional.ofNullable(nextAccount.getAccountNo()).orElse("");// 下一个账户号
82 81 final Date todayDayMinTime = DateUtil.getThisDayMinTime(new Date());
83 82 if (accountType.equals(2) && nextAccountType.equals(2)) {
84 83 if (!Objects.equals(nextAccountNo, "") && !Objects.equals(nextAccountType, -1) &&
85   - (Objects.nonNull(common.getHasFoundFansCnt(nextAccountNo, 2, todayDayMinTime)) &&
  84 + (Objects.nonNull(common.getHasFoundAccountMsg(nextAccountNo, 2, todayDayMinTime)) &&
86 85 Objects.nonNull(common.getHasFoundVideo(nextAccountNo, 2, todayDayMinTime)) &&
87 86 Objects.nonNull(common.getHasFoundLive(nextAccountNo, 2, todayDayMinTime))
88 87 )) {
... ... @@ -97,7 +96,7 @@ public class DataCaptureTask {
97 96 /**
98 97 * 每2分钟执行抓取数据
99 98 */
100   - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
  99 + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
101 100 public void captureLivePlayback() {
102 101 Double durationThreshold = 60d;// 设置直播时长阈值
103 102 final Random random = new Random();
... ... @@ -107,7 +106,7 @@ public class DataCaptureTask {
107 106 .collect(Collectors.toList());// 找到直播时长>60分钟, 失败次数<30的直播数据
108 107 Collection<List<LivePool>> values = withoutPlaybackLive
109 108 .stream()
110   - .collect(Collectors.groupingBy(LivePool::getPhoneNo))
  109 + .collect(Collectors.groupingBy(LivePool::getAccountNo))
111 110 .values()
112 111 .stream()
113 112 .peek(item -> item.sort(Comparator.comparing(LivePool::getGetPlaybackFailTimes)))// 把某个人每条直播按失败次数排序
... ... @@ -115,8 +114,8 @@ public class DataCaptureTask {
115 114 .limit(5)// 一次找5个人的直播
116 115 .collect(Collectors.toList());
117 116 for (List<LivePool> list : values) {// 遍历每个人的直播集合
118   - final String accountNo = list.get(0).getPhoneNo();
119   - Account account = accountDao.findByPhoneNoAndType(accountNo, 1);// 获取账号实体
  117 + final String accountNo = list.get(0).getAccountNo();
  118 + Account account = accountDao.findByAccountNoAndType(accountNo, 1);// 获取账号实体
120 119 final List<JSONObject> playbackMsg = this.getPlaybackMsg(account);// 获取该人的直播回放信息
121 120 boolean flag = false;
122 121 for (LivePool item : list) {
... ... @@ -170,7 +169,7 @@ public class DataCaptureTask {
170 169 if (Objects.nonNull(playbackSearchKey))
171 170 userLivePlayback = kuaiShouCrawl.getUserLivePlayback(playbackSearchKey);
172 171 else
173   - userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getPhoneNo());
  172 + userLivePlayback = kuaiShouCrawl.getUserLivePlayback(account.getAccountNo());
174 173 final List<JSONObject> collect = userLivePlayback
175 174 .stream()
176 175 .filter(item -> {
... ...
src/main/java/cn/fw/freya/utils/RequestUtil.java
  View file @d472478
... ... @@ -696,10 +696,7 @@ public class RequestUtil {
696 696 // 设置通用的请求属性
697 697 conn.setRequestProperty("accept", "*/*");
698 698 conn.setRequestProperty("connection", "Keep-Alive");
699   - conn.setRequestProperty("Host", "cp.kuaishou.com");
700 699 conn.setRequestProperty("Content-Type", "application/json");
701   - conn.setRequestProperty("Content-Length", "84");
702   - conn.setRequestProperty("Cookie", "kuaishou.web.cp.api_ph=c634819dfa9a1da1762bafcf5d195fd101b4;kuaishou.web.cp.api_st=ChZrdWFpc2hvdS53ZWIuY3AuYXBpLnN0EqAB_xuzZVzMjD9DNrreTtCT1cQDRKvB60dNkgWBOQI3mk8_F0WAM1qRKnUtcZYuGH0JTMUntF7JMrs8L7vYLP9tpQ25ppobabbfAcPBqztI7OJdc7MqSwrjanYe6PRZCHnU-tALse-CVxzMFtCl304ekWU1o9fjm7VBQ9EEkC9Q35erQ2RbrnucqQI5E3p_9cd0JE9J8odcLgXOjbWQvT0OuhoSPRRvGtXySX7EnaufBy__uLw_IiA-3ieQkuU_bNGof-EIcBSlmCzad9Ut9hWRS5nEILA9FCgFMAE;_did=web_656080669F3F2217;userId=2079628287;did=web_94f084024029be79aade1f079e2b82e1731a;");
703 700 // 发送POST请求必须设置如下两行
704 701 conn.setDoOutput(true);
705 702 conn.setDoInput(true);
... ...
src/main/resources/NS_sig3Msg.properties 0 → 100644
  View file @d472478
  1 +#
  2 +#Thu Jan 27 15:59:12 CST 2022
  3 +KSAccountNS_sig={}
... ...
src/main/resources/alert.sql 0 → 100644
  View file @d472478
  1 +alter table ACCOUNT drop constraint UKB04BR4OH2RS73RNUSEXU2NLWG;
  2 +alter table ACCOUNT drop constraint UKSOWL8PT6SDTPQPVTXFDP0NPU0;
  3 +alter table ACCOUNT drop column ACCOUNT_NO;
  4 +alter table ACCOUNT alter column PHONE_NO rename to ACCOUNT_NO;
  5 +
  6 +drop index IDXTGFKFY1N98UJQVJ5ME0VR3FSV;
  7 +drop index IDXKWOBILLHNTYYT02FVAQAM5G5;
  8 +alter table COOKIE drop column ACCOUNT_NO;
  9 +alter table COOKIE alter column PHONE_NO rename to ACCOUNT_NO;
  10 +
  11 +alter table LIVE_POOL drop constraint UKANITO7VXALLS54057JU6KJHDF;
  12 +alter table LIVE_POOL drop constraint UKRQ8QS6Q2H9NPW3GV40B86LRXM;
  13 +alter table LIVE_POOL drop column ACCOUNT_NO;
  14 +alter table LIVE_POOL alter column PHONE_NO rename to ACCOUNT_NO;
  15 +
  16 +alter table VIDEO_POOL drop constraint UKL4792GABX9U74HBTHQ0VQM1FG;
  17 +alter table VIDEO_POOL drop constraint UKQ3ROPUUMLOIHN45FGCSUK77G;
  18 +alter table VIDEO_POOL drop column ACCOUNT_NO;
  19 +alter table VIDEO_POOL alter column PHONE_NO rename to ACCOUNT_NO;
... ...