diff --git a/src/main/java/cn/fw/freya/FreyaApplication.java b/src/main/java/cn/fw/freya/FreyaApplication.java index d262b11..6b9a987 100644 --- a/src/main/java/cn/fw/freya/FreyaApplication.java +++ b/src/main/java/cn/fw/freya/FreyaApplication.java @@ -36,12 +36,12 @@ public class FreyaApplication { @Bean(value = "wmyThreadPool") public ThreadPoolExecutor getThreadPool() { return new ThreadPoolExecutor( - 4, - 8, - 15, + 6, + 6, + 30, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(10), + new LinkedBlockingQueue<>(1000), new ThreadFactoryBuilder().setNamePrefix("wmy-thread-pool-").build(), - new ThreadPoolExecutor.AbortPolicy()); + new ThreadPoolExecutor.DiscardPolicy()); } } diff --git a/src/main/java/cn/fw/freya/controller/KSController.java b/src/main/java/cn/fw/freya/controller/KSController.java index b6e169b..e293eb0 100644 --- a/src/main/java/cn/fw/freya/controller/KSController.java +++ b/src/main/java/cn/fw/freya/controller/KSController.java @@ -76,7 +76,7 @@ public class KSController { } /** - * 进入指定账户号的创作者平台首页 + * 获取指定用户指定类型的sig3签名 * * @param accountNo 账户号 */ @@ -86,12 +86,30 @@ public class KSController { } /** - * 获取sigMap + * 设置setSig3Map + * + */ + @GetMapping("/setSig3Map") + public boolean setSig3Map() { + return kuaishouCrawl.setSig3Map(); + } + + /** + * 停止设置setSig3Map + * + */ + @GetMapping("/stopSetSig3Map") + public boolean stopSetSig3Map() { + return kuaishouCrawl.stopSetSig3Map(); + } + + /** + * 获取getSig3Map * */ - @GetMapping("/getSigMap") - public String getSigMap() { - return kuaishouCrawl.getSigMap(); + @GetMapping("/getSig3Map") + public String getSig3Map() { + return kuaishouCrawl.getSig3Map(); } /** diff --git a/src/main/java/cn/fw/freya/dao/AccountDao.java b/src/main/java/cn/fw/freya/dao/AccountDao.java index 2b45c43..429b8e6 100644 --- a/src/main/java/cn/fw/freya/dao/AccountDao.java +++ b/src/main/java/cn/fw/freya/dao/AccountDao.java @@ -46,6 +46,14 @@ public interface AccountDao extends JpaRepository { List findRandomByAndType(Integer type); /** + * 获取所有快手账号 + * + * @return + */ + @Query(nativeQuery = true, value = "select * from `account` where cookies_status = true and `type` = 1;") + List getAllKSAccount(); + + /** * 删除指定类型的账号 * * @param value diff --git a/src/main/java/cn/fw/freya/dao/NsSig3Dao.java b/src/main/java/cn/fw/freya/dao/NsSig3Dao.java new file mode 100644 index 0000000..01dd4dc --- /dev/null +++ b/src/main/java/cn/fw/freya/dao/NsSig3Dao.java @@ -0,0 +1,13 @@ +package cn.fw.freya.dao; + +import cn.fw.freya.model.data.NsSig3; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * @author kurisu + */ +@Repository +public interface NsSig3Dao extends JpaRepository { + +} diff --git a/src/main/java/cn/fw/freya/model/data/NsSig3.java b/src/main/java/cn/fw/freya/model/data/NsSig3.java new file mode 100644 index 0000000..f6bc882 --- /dev/null +++ b/src/main/java/cn/fw/freya/model/data/NsSig3.java @@ -0,0 +1,47 @@ +package cn.fw.freya.model.data; + +import lombok.*; + +import javax.persistence.*; +import java.util.Date; + +/** + * @author unknown + * @date 2021-11-12 10:36 + * @description 签名 + */ +@Data +@ToString +@NoArgsConstructor +@AllArgsConstructor +@Builder(toBuilder = true) +@Entity +@Table(name = "ns_sig3", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type", "sigType"})) +public class NsSig3 { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + /** + * 账户号 + */ + private String accountNo; + /** + * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:哔哩哔哩) + */ + private Integer type; + /** + * 签名类型(1:粉丝, 2:视频, 3:直播) + */ + private Integer sigType; + /** + * 签名值 + */ + @Column(length = 128) + private String sigMsg; + /** + * 创建日期 + */ + @Column(columnDefinition = "date") + @Temporal(TemporalType.DATE) + private Date createDate; +} diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java index eb0a790..46b2c46 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java @@ -38,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; +import javax.annotation.Resource; import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; @@ -45,6 +46,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; @@ -69,15 +71,44 @@ public class KuaiShouCrawl implements CrawlStrategy { private final String playbackBaseUrl = "https://live.kuaishou.com/playback/"; private final ConcurrentHashMap sig3Map = new ConcurrentHashMap<>(); + @Resource(name = "wmyThreadPool") + private ThreadPoolExecutor threadPoolExecutor; + @Override public AccountTypeEnum getType() { return AccountTypeEnum.KS; } /** - * 获取sigMap + * 设置sig3Map + */ + public boolean setSig3Map() { + this.sig3Map.clear(); + final List accountList = accountDao.getAllKSAccount(); + accountList.forEach(item -> + Arrays.stream(DataTypeEnum.values()).forEach(item1 -> + threadPoolExecutor.execute(() -> { + final String accountNo = item.getPhoneNo(); + final Integer typeValue = item1.getValue(); + String key = accountNo + "#" + typeValue; + final String ns_sig3 = this.getNS_sig3(accountNo, typeValue); + if (Objects.nonNull(ns_sig3)) + sig3Map.put(key, ns_sig3); + }) + ) + ); + return true; + } + + public boolean stopSetSig3Map() { + threadPoolExecutor.shutdownNow(); + return true; + } + + /** + * 获取sig3Map */ - public String getSigMap() { + public String getSig3Map() { return JSON.toJSONString(sig3Map); } @@ -684,20 +715,22 @@ public class KuaiShouCrawl implements CrawlStrategy { throw new BusinessException("跳转页面发生异常"); } LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); - if (Objects.equals(type, 1)) { + if (Objects.equals(type, DataTypeEnum.FANS.getValue())) { driver.get("https://cp.kuaishou.com/article/manage/video"); LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5)); if (Objects.equals(targetUrl, driver.getCurrentUrl())) { + this.exitBrowser(accountNo, uuid); return null; } } - if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) { + if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {// 页面未跳转到视频/直播数据页面 + this.exitBrowser(accountNo, uuid); return null; } try { final List responseReceivedEvents = common.processHttpTransferData(driver); for (ResponseReceived item : responseReceivedEvents) { - String str = this.processResp(item, dataUrl); + String str = this.getDataUrl(item, dataUrl); if (StringUtils.hasText(str)) { this.exitBrowser(accountNo, uuid); String[] split = str.split("="); @@ -715,13 +748,13 @@ public class KuaiShouCrawl implements CrawlStrategy { } /** - * 获取日志中的数据 + * 读取http日志获取数据接口全路径地址 * * @param responseReceived 收到的响应 * @param dataUrl 数据接口地址 * @return */ - public String processResp(ResponseReceived responseReceived, String dataUrl) { + public String getDataUrl(ResponseReceived responseReceived, String dataUrl) { String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url"); boolean notStaticFiles = !baseUrl.endsWith(".png") && !baseUrl.endsWith(".jpg") diff --git a/src/main/java/cn/fw/freya/task/DataCaptureTask.java b/src/main/java/cn/fw/freya/task/DataCaptureTask.java index 54c4b4b..b256f56 100644 --- a/src/main/java/cn/fw/freya/task/DataCaptureTask.java +++ b/src/main/java/cn/fw/freya/task/DataCaptureTask.java @@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -51,7 +50,7 @@ public class DataCaptureTask { /** * 每分钟执行多线程同时抓取数据 */ - @Scheduled(cron = "0 0/1 * * * ?") + //@Scheduled(cron = "0 0/1 * * * ?") public void capture() { final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); BlockingQueue queue = threadPoolExecutor.getQueue();// 获取工作队列 @@ -97,7 +96,7 @@ public class DataCaptureTask { /** * 每2分钟执行抓取数据 */ - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) public void captureLivePlayback() { Double durationThreshold = 60d;// 设置直播时长阈值 final Random random = new Random();