Commit 6c0810cc9815b372555d5ff81338de0112be43df
1 parent
8f9b88ab
2022年6月29日11:36:04 增加获取__NS_sig3信息接口
Showing
7 changed files
with
138 additions
and
20 deletions
src/main/java/cn/fw/freya/FreyaApplication.java
... | ... | @@ -36,12 +36,12 @@ public class FreyaApplication { |
36 | 36 | @Bean(value = "wmyThreadPool") |
37 | 37 | public ThreadPoolExecutor getThreadPool() { |
38 | 38 | return new ThreadPoolExecutor( |
39 | - 4, | |
40 | - 8, | |
41 | - 15, | |
39 | + 6, | |
40 | + 6, | |
41 | + 30, | |
42 | 42 | TimeUnit.SECONDS, |
43 | - new LinkedBlockingQueue<>(10), | |
43 | + new LinkedBlockingQueue<>(1000), | |
44 | 44 | new ThreadFactoryBuilder().setNamePrefix("wmy-thread-pool-").build(), |
45 | - new ThreadPoolExecutor.AbortPolicy()); | |
45 | + new ThreadPoolExecutor.DiscardPolicy()); | |
46 | 46 | } |
47 | 47 | } | ... | ... |
src/main/java/cn/fw/freya/controller/KSController.java
... | ... | @@ -76,7 +76,7 @@ public class KSController { |
76 | 76 | } |
77 | 77 | |
78 | 78 | /** |
79 | - * 进入指定账户号的创作者平台首页 | |
79 | + * 获取指定用户指定类型的sig3签名 | |
80 | 80 | * |
81 | 81 | * @param accountNo 账户号 |
82 | 82 | */ |
... | ... | @@ -86,12 +86,30 @@ public class KSController { |
86 | 86 | } |
87 | 87 | |
88 | 88 | /** |
89 | - * 获取sigMap | |
89 | + * 设置setSig3Map | |
90 | + * | |
91 | + */ | |
92 | + @GetMapping("/setSig3Map") | |
93 | + public boolean setSig3Map() { | |
94 | + return kuaishouCrawl.setSig3Map(); | |
95 | + } | |
96 | + | |
97 | + /** | |
98 | + * 停止设置setSig3Map | |
99 | + * | |
100 | + */ | |
101 | + @GetMapping("/stopSetSig3Map") | |
102 | + public boolean stopSetSig3Map() { | |
103 | + return kuaishouCrawl.stopSetSig3Map(); | |
104 | + } | |
105 | + | |
106 | + /** | |
107 | + * 获取getSig3Map | |
90 | 108 | * |
91 | 109 | */ |
92 | - @GetMapping("/getSigMap") | |
93 | - public String getSigMap() { | |
94 | - return kuaishouCrawl.getSigMap(); | |
110 | + @GetMapping("/getSig3Map") | |
111 | + public String getSig3Map() { | |
112 | + return kuaishouCrawl.getSig3Map(); | |
95 | 113 | } |
96 | 114 | |
97 | 115 | /** | ... | ... |
src/main/java/cn/fw/freya/dao/AccountDao.java
... | ... | @@ -46,6 +46,14 @@ public interface AccountDao extends JpaRepository<Account, Long> { |
46 | 46 | List<Account> findRandomByAndType(Integer type); |
47 | 47 | |
48 | 48 | /** |
49 | + * 获取所有快手账号 | |
50 | + * | |
51 | + * @return | |
52 | + */ | |
53 | + @Query(nativeQuery = true, value = "select * from `account` where cookies_status = true and `type` = 1;") | |
54 | + List<Account> getAllKSAccount(); | |
55 | + | |
56 | + /** | |
49 | 57 | * 删除指定类型的账号 |
50 | 58 | * |
51 | 59 | * @param value | ... | ... |
src/main/java/cn/fw/freya/dao/NsSig3Dao.java
0 → 100644
1 | +package cn.fw.freya.dao; | |
2 | + | |
3 | +import cn.fw.freya.model.data.NsSig3; | |
4 | +import org.springframework.data.jpa.repository.JpaRepository; | |
5 | +import org.springframework.stereotype.Repository; | |
6 | + | |
7 | +/** | |
8 | + * @author kurisu | |
9 | + */ | |
10 | +@Repository | |
11 | +public interface NsSig3Dao extends JpaRepository<NsSig3, Long> { | |
12 | + | |
13 | +} | ... | ... |
src/main/java/cn/fw/freya/model/data/NsSig3.java
0 → 100644
1 | +package cn.fw.freya.model.data; | |
2 | + | |
3 | +import lombok.*; | |
4 | + | |
5 | +import javax.persistence.*; | |
6 | +import java.util.Date; | |
7 | + | |
8 | +/** | |
9 | + * @author unknown | |
10 | + * @date 2021-11-12 10:36 | |
11 | + * @description 签名 | |
12 | + */ | |
13 | +@Data | |
14 | +@ToString | |
15 | +@NoArgsConstructor | |
16 | +@AllArgsConstructor | |
17 | +@Builder(toBuilder = true) | |
18 | +@Entity | |
19 | +@Table(name = "ns_sig3", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type", "sigType"})) | |
20 | +public class NsSig3 { | |
21 | + @Id | |
22 | + @GeneratedValue(strategy = GenerationType.IDENTITY) | |
23 | + private Long id; | |
24 | + /** | |
25 | + * 账户号 | |
26 | + */ | |
27 | + private String accountNo; | |
28 | + /** | |
29 | + * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:哔哩哔哩) | |
30 | + */ | |
31 | + private Integer type; | |
32 | + /** | |
33 | + * 签名类型(1:粉丝, 2:视频, 3:直播) | |
34 | + */ | |
35 | + private Integer sigType; | |
36 | + /** | |
37 | + * 签名值 | |
38 | + */ | |
39 | + @Column(length = 128) | |
40 | + private String sigMsg; | |
41 | + /** | |
42 | + * 创建日期 | |
43 | + */ | |
44 | + @Column(columnDefinition = "date") | |
45 | + @Temporal(TemporalType.DATE) | |
46 | + private Date createDate; | |
47 | +} | ... | ... |
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
... | ... | @@ -38,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional; |
38 | 38 | import org.springframework.util.CollectionUtils; |
39 | 39 | import org.springframework.util.StringUtils; |
40 | 40 | |
41 | +import javax.annotation.Resource; | |
41 | 42 | import java.io.IOException; |
42 | 43 | import java.math.BigDecimal; |
43 | 44 | import java.math.RoundingMode; |
... | ... | @@ -45,6 +46,7 @@ import java.time.LocalDate; |
45 | 46 | import java.time.LocalDateTime; |
46 | 47 | import java.util.*; |
47 | 48 | import java.util.concurrent.ConcurrentHashMap; |
49 | +import java.util.concurrent.ThreadPoolExecutor; | |
48 | 50 | import java.util.concurrent.TimeUnit; |
49 | 51 | import java.util.concurrent.locks.LockSupport; |
50 | 52 | import java.util.stream.Collectors; |
... | ... | @@ -69,15 +71,44 @@ public class KuaiShouCrawl implements CrawlStrategy { |
69 | 71 | private final String playbackBaseUrl = "https://live.kuaishou.com/playback/"; |
70 | 72 | private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>(); |
71 | 73 | |
74 | + @Resource(name = "wmyThreadPool") | |
75 | + private ThreadPoolExecutor threadPoolExecutor; | |
76 | + | |
72 | 77 | @Override |
73 | 78 | public AccountTypeEnum getType() { |
74 | 79 | return AccountTypeEnum.KS; |
75 | 80 | } |
76 | 81 | |
77 | 82 | /** |
78 | - * 获取sigMap | |
83 | + * 设置sig3Map | |
84 | + */ | |
85 | + public boolean setSig3Map() { | |
86 | + this.sig3Map.clear(); | |
87 | + final List<Account> accountList = accountDao.getAllKSAccount(); | |
88 | + accountList.forEach(item -> | |
89 | + Arrays.stream(DataTypeEnum.values()).forEach(item1 -> | |
90 | + threadPoolExecutor.execute(() -> { | |
91 | + final String accountNo = item.getPhoneNo(); | |
92 | + final Integer typeValue = item1.getValue(); | |
93 | + String key = accountNo + "#" + typeValue; | |
94 | + final String ns_sig3 = this.getNS_sig3(accountNo, typeValue); | |
95 | + if (Objects.nonNull(ns_sig3)) | |
96 | + sig3Map.put(key, ns_sig3); | |
97 | + }) | |
98 | + ) | |
99 | + ); | |
100 | + return true; | |
101 | + } | |
102 | + | |
103 | + public boolean stopSetSig3Map() { | |
104 | + threadPoolExecutor.shutdownNow(); | |
105 | + return true; | |
106 | + } | |
107 | + | |
108 | + /** | |
109 | + * 获取sig3Map | |
79 | 110 | */ |
80 | - public String getSigMap() { | |
111 | + public String getSig3Map() { | |
81 | 112 | return JSON.toJSONString(sig3Map); |
82 | 113 | } |
83 | 114 | |
... | ... | @@ -684,20 +715,22 @@ public class KuaiShouCrawl implements CrawlStrategy { |
684 | 715 | throw new BusinessException("跳转页面发生异常"); |
685 | 716 | } |
686 | 717 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); |
687 | - if (Objects.equals(type, 1)) { | |
718 | + if (Objects.equals(type, DataTypeEnum.FANS.getValue())) { | |
688 | 719 | driver.get("https://cp.kuaishou.com/article/manage/video"); |
689 | 720 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); |
690 | 721 | if (Objects.equals(targetUrl, driver.getCurrentUrl())) { |
722 | + this.exitBrowser(accountNo, uuid); | |
691 | 723 | return null; |
692 | 724 | } |
693 | 725 | } |
694 | - if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) { | |
726 | + if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {// 页面未跳转到视频/直播数据页面 | |
727 | + this.exitBrowser(accountNo, uuid); | |
695 | 728 | return null; |
696 | 729 | } |
697 | 730 | try { |
698 | 731 | final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver); |
699 | 732 | for (ResponseReceived item : responseReceivedEvents) { |
700 | - String str = this.processResp(item, dataUrl); | |
733 | + String str = this.getDataUrl(item, dataUrl); | |
701 | 734 | if (StringUtils.hasText(str)) { |
702 | 735 | this.exitBrowser(accountNo, uuid); |
703 | 736 | String[] split = str.split("="); |
... | ... | @@ -715,13 +748,13 @@ public class KuaiShouCrawl implements CrawlStrategy { |
715 | 748 | } |
716 | 749 | |
717 | 750 | /** |
718 | - * 获取日志中的数据 | |
751 | + * 读取http日志获取数据接口全路径地址 | |
719 | 752 | * |
720 | 753 | * @param responseReceived 收到的响应 |
721 | 754 | * @param dataUrl 数据接口地址 |
722 | 755 | * @return |
723 | 756 | */ |
724 | - public String processResp(ResponseReceived responseReceived, String dataUrl) { | |
757 | + public String getDataUrl(ResponseReceived responseReceived, String dataUrl) { | |
725 | 758 | String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url"); |
726 | 759 | boolean notStaticFiles = !baseUrl.endsWith(".png") |
727 | 760 | && !baseUrl.endsWith(".jpg") | ... | ... |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
... | ... | @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; |
16 | 16 | import lombok.RequiredArgsConstructor; |
17 | 17 | import lombok.extern.slf4j.Slf4j; |
18 | 18 | import org.springframework.context.ApplicationEventPublisher; |
19 | -import org.springframework.scheduling.annotation.Scheduled; | |
20 | 19 | import org.springframework.stereotype.Component; |
21 | 20 | import org.springframework.util.CollectionUtils; |
22 | 21 | |
... | ... | @@ -51,7 +50,7 @@ public class DataCaptureTask { |
51 | 50 | /** |
52 | 51 | * 每分钟执行多线程同时抓取数据 |
53 | 52 | */ |
54 | - @Scheduled(cron = "0 0/1 * * * ?") | |
53 | + //@Scheduled(cron = "0 0/1 * * * ?") | |
55 | 54 | public void capture() { |
56 | 55 | final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); |
57 | 56 | BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 |
... | ... | @@ -97,7 +96,7 @@ public class DataCaptureTask { |
97 | 96 | /** |
98 | 97 | * 每2分钟执行抓取数据 |
99 | 98 | */ |
100 | - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) | |
99 | + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) | |
101 | 100 | public void captureLivePlayback() { |
102 | 101 | Double durationThreshold = 60d;// 设置直播时长阈值 |
103 | 102 | final Random random = new Random(); | ... | ... |