2022年6月29日11:36:04 增加获取__NS_sig3信息接口

王明元
1 parent 8f9b88ab
Showing 7 changed files with 138 additions and 20 deletions
src/main/java/cn/fw/freya/FreyaApplication.java
src/main/java/cn/fw/freya/controller/KSController.java
src/main/java/cn/fw/freya/dao/AccountDao.java
src/main/java/cn/fw/freya/dao/NsSig3Dao.java
src/main/java/cn/fw/freya/model/data/NsSig3.java
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
src/main/java/cn/fw/freya/task/DataCaptureTask.java
@@ -36,12 +36,12 @@ public class FreyaApplication {
     @Bean(value = "wmyThreadPool")
     public ThreadPoolExecutor getThreadPool() {
         return new ThreadPoolExecutor(
-                4,
-                8,
-                15,
+                6,
+                6,
+                30,
                 TimeUnit.SECONDS,
-                new LinkedBlockingQueue<>(10),
+                new LinkedBlockingQueue<>(1000),
                 new ThreadFactoryBuilder().setNamePrefix("wmy-thread-pool-").build(),
-                new ThreadPoolExecutor.AbortPolicy());
+                new ThreadPoolExecutor.DiscardPolicy());
     }
 }
@@ -76,7 +76,7 @@ public class KSController {
     }
  
     /**
-     * 进入指定账户号的创作者平台首页
+     * 获取指定用户指定类型的sig3签名
      *
      * @param accountNo 账户号
      */
@@ -86,12 +86,30 @@ public class KSController {
     }
  
     /**
-     * 获取sigMap
+     * 设置setSig3Map
+     *
+     */
+    @GetMapping("/setSig3Map")
+    public boolean setSig3Map() {
+        return kuaishouCrawl.setSig3Map();
+    }
+
+    /**
+     * 停止设置setSig3Map
+     *
+     */
+    @GetMapping("/stopSetSig3Map")
+    public boolean stopSetSig3Map() {
+        return kuaishouCrawl.stopSetSig3Map();
+    }
+
+    /**
+     * 获取getSig3Map
      *
      */
-    @GetMapping("/getSigMap")
-    public String getSigMap() {
-        return kuaishouCrawl.getSigMap();
+    @GetMapping("/getSig3Map")
+    public String getSig3Map() {
+        return kuaishouCrawl.getSig3Map();
     }
  
     /**
@@ -46,6 +46,14 @@ public interface AccountDao extends JpaRepository&lt;Account, Long&gt; {
     List<Account> findRandomByAndType(Integer type);
  
     /**
+     * 获取所有快手账号
+     *
+     * @return
+     */
+    @Query(nativeQuery = true, value = "select * from `account` where cookies_status = true and `type` = 1;")
+    List<Account> getAllKSAccount();
+
+    /**
      * 删除指定类型的账号
      *
      * @param value
+package cn.fw.freya.dao;
+
+import cn.fw.freya.model.data.NsSig3;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+/**
+ * @author kurisu
+ */
+@Repository
+public interface NsSig3Dao extends JpaRepository<NsSig3, Long> {
+
+}
+package cn.fw.freya.model.data;
+
+import lombok.*;
+
+import javax.persistence.*;
+import java.util.Date;
+
+/**
+ * @author unknown
+ * @date 2021-11-12 10:36
+ * @description 签名
+ */
+@Data
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder(toBuilder = true)
+@Entity
+@Table(name = "ns_sig3", uniqueConstraints = @UniqueConstraint(columnNames = {"accountNo", "type", "sigType"}))
+public class NsSig3 {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+    /**
+     * 账户号
+     */
+    private String accountNo;
+    /**
+     * 账户类型 (1:快手, 2:抖音, 3:懂车帝, 4:哔哩哔哩)
+     */
+    private Integer type;
+    /**
+     * 签名类型(1:粉丝, 2:视频, 3:直播)
+     */
+    private Integer sigType;
+    /**
+     * 签名值
+     */
+    @Column(length = 128)
+    private String sigMsg;
+    /**
+     * 创建日期
+     */
+    @Column(columnDefinition = "date")
+    @Temporal(TemporalType.DATE)
+    private Date createDate;
+}
@@ -38,6 +38,7 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
  
+import javax.annotation.Resource;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
@@ -45,6 +46,7 @@ import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.LockSupport;
 import java.util.stream.Collectors;
@@ -69,15 +71,44 @@ public class KuaiShouCrawl implements CrawlStrategy {
     private final String playbackBaseUrl = "https://live.kuaishou.com/playback/";
     private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>();
  
+    @Resource(name = "wmyThreadPool")
+    private ThreadPoolExecutor threadPoolExecutor;
+
     @Override
     public AccountTypeEnum getType() {
         return AccountTypeEnum.KS;
     }
  
     /**
-     * 获取sigMap
+     * 设置sig3Map
+     */
+    public boolean setSig3Map() {
+        this.sig3Map.clear();
+        final List<Account> accountList = accountDao.getAllKSAccount();
+        accountList.forEach(item ->
+                Arrays.stream(DataTypeEnum.values()).forEach(item1 ->
+                        threadPoolExecutor.execute(() -> {
+                            final String accountNo = item.getPhoneNo();
+                            final Integer typeValue = item1.getValue();
+                            String key = accountNo + "#" + typeValue;
+                            final String ns_sig3 = this.getNS_sig3(accountNo, typeValue);
+                            if (Objects.nonNull(ns_sig3))
+                                sig3Map.put(key, ns_sig3);
+                        })
+                )
+        );
+        return true;
+    }
+
+    public boolean stopSetSig3Map() {
+        threadPoolExecutor.shutdownNow();
+        return true;
+    }
+
+    /**
+     * 获取sig3Map
      */
-    public String getSigMap() {
+    public String getSig3Map() {
         return JSON.toJSONString(sig3Map);
     }
  
@@ -684,20 +715,22 @@ public class KuaiShouCrawl implements CrawlStrategy {
             throw new BusinessException("跳转页面发生异常");
         }
         LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
-        if (Objects.equals(type, 1)) {
+        if (Objects.equals(type, DataTypeEnum.FANS.getValue())) {
             driver.get("https://cp.kuaishou.com/article/manage/video");
             LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
             if (Objects.equals(targetUrl, driver.getCurrentUrl())) {
+                this.exitBrowser(accountNo, uuid);
                 return null;
             }
         }
-        if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {
+        if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {// 页面未跳转到视频/直播数据页面
+            this.exitBrowser(accountNo, uuid);
             return null;
         }
         try {
             final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
             for (ResponseReceived item : responseReceivedEvents) {
-                String str = this.processResp(item, dataUrl);
+                String str = this.getDataUrl(item, dataUrl);
                 if (StringUtils.hasText(str)) {
                     this.exitBrowser(accountNo, uuid);
                     String[] split = str.split("=");
@@ -715,13 +748,13 @@ public class KuaiShouCrawl implements CrawlStrategy {
     }
  
     /**
-     * 获取日志中的数据
+     * 读取http日志获取数据接口全路径地址
      *
      * @param responseReceived 收到的响应
      * @param dataUrl          数据接口地址
      * @return
      */
-    public String processResp(ResponseReceived responseReceived, String dataUrl) {
+    public String getDataUrl(ResponseReceived responseReceived, String dataUrl) {
         String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url");
         boolean notStaticFiles = !baseUrl.endsWith(".png")
                 && !baseUrl.endsWith(".jpg")
@@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONObject;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.ApplicationEventPublisher;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
  
@@ -51,7 +50,7 @@ public class DataCaptureTask {
     /**
      * 每分钟执行多线程同时抓取数据
      */
-    @Scheduled(cron = "0 0/1 * * * ?")
+    //@Scheduled(cron = "0 0/1 * * * ?")
     public void capture() {
         final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool();
         BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列
@@ -97,7 +96,7 @@ public class DataCaptureTask {
     /**
      * 每2分钟执行抓取数据
      */
-    @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
+    //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
     public void captureLivePlayback() {
         Double durationThreshold = 60d;// 设置直播时长阈值
         final Random random = new Random();