Commit 6c0810cc9815b372555d5ff81338de0112be43df
1 parent
8f9b88ab
2022年6月29日11:36:04 增加获取__NS_sig3信息接口
Showing
7 changed files
with
138 additions
and
20 deletions
src/main/java/cn/fw/freya/FreyaApplication.java
@@ -36,12 +36,12 @@ public class FreyaApplication { | @@ -36,12 +36,12 @@ public class FreyaApplication { | ||
36 | @Bean(value = "wmyThreadPool") | 36 | @Bean(value = "wmyThreadPool") |
37 | public ThreadPoolExecutor getThreadPool() { | 37 | public ThreadPoolExecutor getThreadPool() { |
38 | return new ThreadPoolExecutor( | 38 | return new ThreadPoolExecutor( |
39 | - 4, | ||
40 | - 8, | ||
41 | - 15, | 39 | + 6, |
40 | + 6, | ||
41 | + 30, | ||
42 | TimeUnit.SECONDS, | 42 | TimeUnit.SECONDS, |
43 | - new LinkedBlockingQueue<>(10), | 43 | + new LinkedBlockingQueue<>(1000), |
44 | new ThreadFactoryBuilder().setNamePrefix("wmy-thread-pool-").build(), | 44 | new ThreadFactoryBuilder().setNamePrefix("wmy-thread-pool-").build(), |
45 | - new ThreadPoolExecutor.AbortPolicy()); | 45 | + new ThreadPoolExecutor.DiscardPolicy()); |
46 | } | 46 | } |
47 | } | 47 | } |
src/main/java/cn/fw/freya/controller/KSController.java
@@ -76,7 +76,7 @@ public class KSController { | @@ -76,7 +76,7 @@ public class KSController { | ||
76 | } | 76 | } |
77 | 77 | ||
78 | /** | 78 | /** |
79 | - * 进入指定账户号的创作者平台首页 | 79 | + * 获取指定用户指定类型的sig3签名 |
80 | * | 80 | * |
81 | * @param accountNo 账户号 | 81 | * @param accountNo 账户号 |
82 | */ | 82 | */ |
@@ -86,12 +86,30 @@ public class KSController { | @@ -86,12 +86,30 @@ public class KSController { | ||
86 | } | 86 | } |
87 | 87 | ||
88 | /** | 88 | /** |
89 | - * 获取sigMap | 89 | + * 设置setSig3Map |
90 | + * | ||
91 | + */ | ||
92 | + @GetMapping("/setSig3Map") | ||
93 | + public boolean setSig3Map() { | ||
94 | + return kuaishouCrawl.setSig3Map(); | ||
95 | + } | ||
96 | + | ||
97 | + /** | ||
98 | + * 停止设置setSig3Map | ||
99 | + * | ||
100 | + */ | ||
101 | + @GetMapping("/stopSetSig3Map") | ||
102 | + public boolean stopSetSig3Map() { | ||
103 | + return kuaishouCrawl.stopSetSig3Map(); | ||
104 | + } | ||
105 | + | ||
106 | + /** | ||
107 | + * 获取getSig3Map | ||
90 | * | 108 | * |
91 | */ | 109 | */ |
92 | - @GetMapping("/getSigMap") | ||
93 | - public String getSigMap() { | ||
94 | - return kuaishouCrawl.getSigMap(); | 110 | + @GetMapping("/getSig3Map") |
111 | + public String getSig3Map() { | ||
112 | + return kuaishouCrawl.getSig3Map(); | ||
95 | } | 113 | } |
96 | 114 | ||
97 | /** | 115 | /** |
src/main/java/cn/fw/freya/dao/AccountDao.java
@@ -46,6 +46,14 @@ public interface AccountDao extends JpaRepository<Account, Long> { | @@ -46,6 +46,14 @@ public interface AccountDao extends JpaRepository<Account, Long> { | ||
46 | List<Account> findRandomByAndType(Integer type); | 46 | List<Account> findRandomByAndType(Integer type); |
47 | 47 | ||
48 | /** | 48 | /** |
49 | + * 获取所有快手账号 | ||
50 | + * | ||
51 | + * @return | ||
52 | + */ | ||
53 | + @Query(nativeQuery = true, value = "select * from `account` where cookies_status = true and `type` = 1;") | ||
54 | + List<Account> getAllKSAccount(); | ||
55 | + | ||
56 | + /** | ||
49 | * 删除指定类型的账号 | 57 | * 删除指定类型的账号 |
50 | * | 58 | * |
51 | * @param value | 59 | * @param value |
src/main/java/cn/fw/freya/dao/NsSig3Dao.java
0 → 100644
1 | +package cn.fw.freya.dao; | ||
2 | + | ||
3 | +import cn.fw.freya.model.data.NsSig3; | ||
4 | +import org.springframework.data.jpa.repository.JpaRepository; | ||
5 | +import org.springframework.stereotype.Repository; | ||
6 | + | ||
7 | +/** | ||
8 | + * @author kurisu | ||
9 | + */ | ||
10 | +@Repository | ||
11 | +public interface NsSig3Dao extends JpaRepository<NsSig3, Long> { | ||
12 | + | ||
13 | +} |
src/main/java/cn/fw/freya/model/data/NsSig3.java
0 → 100644
1 | +package cn.fw.freya.model.data; | ||
2 | + | ||
3 | +import lombok.*; | ||
4 | + | ||
5 | +import javax.persistence.*; | ||
6 | +import java.util.Date; | ||
7 | + | ||
8 | +/** | ||
9 | + * @author unknown | ||
10 | + * @date 2021-11-12 10:36 | ||
11 | + * @description 签名 | ||
12 | + */ | ||
13 | +@Data | ||
14 | +@ToString | ||
15 | +@NoArgsConstructor | ||
16 | +@AllArgsConstructor | ||
17 | +@Builder(toBuilder = true) | ||
18 | +@Entity | ||
19 | +@Table(name = "ns_sig3", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type", "sigType"})) | ||
20 | +public class NsSig3 { | ||
21 | + @Id | ||
22 | + @GeneratedValue(strategy = GenerationType.IDENTITY) | ||
23 | + private Long id; | ||
24 | + /** | ||
25 | + * 账户号 | ||
26 | + */ | ||
27 | + private String accountNo; | ||
28 | + /** | ||
29 | + * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:哔哩哔哩) | ||
30 | + */ | ||
31 | + private Integer type; | ||
32 | + /** | ||
33 | + * 签名类型(1:粉丝, 2:视频, 3:直播) | ||
34 | + */ | ||
35 | + private Integer sigType; | ||
36 | + /** | ||
37 | + * 签名值 | ||
38 | + */ | ||
39 | + @Column(length = 128) | ||
40 | + private String sigMsg; | ||
41 | + /** | ||
42 | + * 创建日期 | ||
43 | + */ | ||
44 | + @Column(columnDefinition = "date") | ||
45 | + @Temporal(TemporalType.DATE) | ||
46 | + private Date createDate; | ||
47 | +} |
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
@@ -38,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional; | @@ -38,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional; | ||
38 | import org.springframework.util.CollectionUtils; | 38 | import org.springframework.util.CollectionUtils; |
39 | import org.springframework.util.StringUtils; | 39 | import org.springframework.util.StringUtils; |
40 | 40 | ||
41 | +import javax.annotation.Resource; | ||
41 | import java.io.IOException; | 42 | import java.io.IOException; |
42 | import java.math.BigDecimal; | 43 | import java.math.BigDecimal; |
43 | import java.math.RoundingMode; | 44 | import java.math.RoundingMode; |
@@ -45,6 +46,7 @@ import java.time.LocalDate; | @@ -45,6 +46,7 @@ import java.time.LocalDate; | ||
45 | import java.time.LocalDateTime; | 46 | import java.time.LocalDateTime; |
46 | import java.util.*; | 47 | import java.util.*; |
47 | import java.util.concurrent.ConcurrentHashMap; | 48 | import java.util.concurrent.ConcurrentHashMap; |
49 | +import java.util.concurrent.ThreadPoolExecutor; | ||
48 | import java.util.concurrent.TimeUnit; | 50 | import java.util.concurrent.TimeUnit; |
49 | import java.util.concurrent.locks.LockSupport; | 51 | import java.util.concurrent.locks.LockSupport; |
50 | import java.util.stream.Collectors; | 52 | import java.util.stream.Collectors; |
@@ -69,15 +71,44 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -69,15 +71,44 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
69 | private final String playbackBaseUrl = "https://live.kuaishou.com/playback/"; | 71 | private final String playbackBaseUrl = "https://live.kuaishou.com/playback/"; |
70 | private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>(); | 72 | private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>(); |
71 | 73 | ||
74 | + @Resource(name = "wmyThreadPool") | ||
75 | + private ThreadPoolExecutor threadPoolExecutor; | ||
76 | + | ||
72 | @Override | 77 | @Override |
73 | public AccountTypeEnum getType() { | 78 | public AccountTypeEnum getType() { |
74 | return AccountTypeEnum.KS; | 79 | return AccountTypeEnum.KS; |
75 | } | 80 | } |
76 | 81 | ||
77 | /** | 82 | /** |
78 | - * 获取sigMap | 83 | + * 设置sig3Map |
84 | + */ | ||
85 | + public boolean setSig3Map() { | ||
86 | + this.sig3Map.clear(); | ||
87 | + final List<Account> accountList = accountDao.getAllKSAccount(); | ||
88 | + accountList.forEach(item -> | ||
89 | + Arrays.stream(DataTypeEnum.values()).forEach(item1 -> | ||
90 | + threadPoolExecutor.execute(() -> { | ||
91 | + final String accountNo = item.getPhoneNo(); | ||
92 | + final Integer typeValue = item1.getValue(); | ||
93 | + String key = accountNo + "#" + typeValue; | ||
94 | + final String ns_sig3 = this.getNS_sig3(accountNo, typeValue); | ||
95 | + if (Objects.nonNull(ns_sig3)) | ||
96 | + sig3Map.put(key, ns_sig3); | ||
97 | + }) | ||
98 | + ) | ||
99 | + ); | ||
100 | + return true; | ||
101 | + } | ||
102 | + | ||
103 | + public boolean stopSetSig3Map() { | ||
104 | + threadPoolExecutor.shutdownNow(); | ||
105 | + return true; | ||
106 | + } | ||
107 | + | ||
108 | + /** | ||
109 | + * 获取sig3Map | ||
79 | */ | 110 | */ |
80 | - public String getSigMap() { | 111 | + public String getSig3Map() { |
81 | return JSON.toJSONString(sig3Map); | 112 | return JSON.toJSONString(sig3Map); |
82 | } | 113 | } |
83 | 114 | ||
@@ -684,20 +715,22 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -684,20 +715,22 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
684 | throw new BusinessException("跳转页面发生异常"); | 715 | throw new BusinessException("跳转页面发生异常"); |
685 | } | 716 | } |
686 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); | 717 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); |
687 | - if (Objects.equals(type, 1)) { | 718 | + if (Objects.equals(type, DataTypeEnum.FANS.getValue())) { |
688 | driver.get("https://cp.kuaishou.com/article/manage/video"); | 719 | driver.get("https://cp.kuaishou.com/article/manage/video"); |
689 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); | 720 | LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); |
690 | if (Objects.equals(targetUrl, driver.getCurrentUrl())) { | 721 | if (Objects.equals(targetUrl, driver.getCurrentUrl())) { |
722 | + this.exitBrowser(accountNo, uuid); | ||
691 | return null; | 723 | return null; |
692 | } | 724 | } |
693 | } | 725 | } |
694 | - if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) { | 726 | + if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {// 页面未跳转到视频/直播数据页面 |
727 | + this.exitBrowser(accountNo, uuid); | ||
695 | return null; | 728 | return null; |
696 | } | 729 | } |
697 | try { | 730 | try { |
698 | final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver); | 731 | final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver); |
699 | for (ResponseReceived item : responseReceivedEvents) { | 732 | for (ResponseReceived item : responseReceivedEvents) { |
700 | - String str = this.processResp(item, dataUrl); | 733 | + String str = this.getDataUrl(item, dataUrl); |
701 | if (StringUtils.hasText(str)) { | 734 | if (StringUtils.hasText(str)) { |
702 | this.exitBrowser(accountNo, uuid); | 735 | this.exitBrowser(accountNo, uuid); |
703 | String[] split = str.split("="); | 736 | String[] split = str.split("="); |
@@ -715,13 +748,13 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -715,13 +748,13 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
715 | } | 748 | } |
716 | 749 | ||
717 | /** | 750 | /** |
718 | - * 获取日志中的数据 | 751 | + * 读取http日志获取数据接口全路径地址 |
719 | * | 752 | * |
720 | * @param responseReceived 收到的响应 | 753 | * @param responseReceived 收到的响应 |
721 | * @param dataUrl 数据接口地址 | 754 | * @param dataUrl 数据接口地址 |
722 | * @return | 755 | * @return |
723 | */ | 756 | */ |
724 | - public String processResp(ResponseReceived responseReceived, String dataUrl) { | 757 | + public String getDataUrl(ResponseReceived responseReceived, String dataUrl) { |
725 | String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url"); | 758 | String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url"); |
726 | boolean notStaticFiles = !baseUrl.endsWith(".png") | 759 | boolean notStaticFiles = !baseUrl.endsWith(".png") |
727 | && !baseUrl.endsWith(".jpg") | 760 | && !baseUrl.endsWith(".jpg") |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
@@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; | @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; | ||
16 | import lombok.RequiredArgsConstructor; | 16 | import lombok.RequiredArgsConstructor; |
17 | import lombok.extern.slf4j.Slf4j; | 17 | import lombok.extern.slf4j.Slf4j; |
18 | import org.springframework.context.ApplicationEventPublisher; | 18 | import org.springframework.context.ApplicationEventPublisher; |
19 | -import org.springframework.scheduling.annotation.Scheduled; | ||
20 | import org.springframework.stereotype.Component; | 19 | import org.springframework.stereotype.Component; |
21 | import org.springframework.util.CollectionUtils; | 20 | import org.springframework.util.CollectionUtils; |
22 | 21 | ||
@@ -51,7 +50,7 @@ public class DataCaptureTask { | @@ -51,7 +50,7 @@ public class DataCaptureTask { | ||
51 | /** | 50 | /** |
52 | * 每分钟执行多线程同时抓取数据 | 51 | * 每分钟执行多线程同时抓取数据 |
53 | */ | 52 | */ |
54 | - @Scheduled(cron = "0 0/1 * * * ?") | 53 | + //@Scheduled(cron = "0 0/1 * * * ?") |
55 | public void capture() { | 54 | public void capture() { |
56 | final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); | 55 | final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); |
57 | BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 | 56 | BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 |
@@ -97,7 +96,7 @@ public class DataCaptureTask { | @@ -97,7 +96,7 @@ public class DataCaptureTask { | ||
97 | /** | 96 | /** |
98 | * 每2分钟执行抓取数据 | 97 | * 每2分钟执行抓取数据 |
99 | */ | 98 | */ |
100 | - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) | 99 | + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) |
101 | public void captureLivePlayback() { | 100 | public void captureLivePlayback() { |
102 | Double durationThreshold = 60d;// 设置直播时长阈值 | 101 | Double durationThreshold = 60d;// 设置直播时长阈值 |
103 | final Random random = new Random(); | 102 | final Random random = new Random(); |