third / fw-freya

Commit 5736d4ca986ec0028cc5def4976b6c7562614771

Authored by 王明元
1 parent 44e2b8d5

2023年1月9日14:54:38 修复直播回放无法获取问题

Inline Side-by-side
Showing 5 changed files with 79 additions and 19 deletions
src/main/java/cn/fw/freya/controller/KSController.java
  View file @5736d4c
... ... @@ -11,8 +11,6 @@ import org.springframework.web.bind.annotation.*;
11 11  
12 12 import javax.servlet.http.HttpServletRequest;
13 13 import java.io.IOException;
14   -import java.math.BigDecimal;
15   -import java.math.RoundingMode;
16 14 import java.text.SimpleDateFormat;
17 15 import java.util.Arrays;
18 16 import java.util.Date;
... ... @@ -143,10 +141,14 @@ public class KSController {
143 141 return kuaishouCrawl.getUserLivePlayback(accountNo).stream()
144 142 .map(item -> {
145 143 JSONObject obj = (JSONObject) item;
146   - obj.put("duration", BigDecimal.valueOf(obj.getLong("duration")).divide(BigDecimal.valueOf(60), 2, RoundingMode.HALF_UP));
147   - Arrays.asList("createTime").forEach(item1 -> obj.put(item1, sdf.format(new Date(obj.getLong(item1)))));
  144 + //obj.put("duration", BigDecimal.valueOf(obj.getLong("duration")).divide(BigDecimal.valueOf(60), 2, RoundingMode.HALF_UP));
  145 + Arrays.asList("createTime").forEach(item1 ->
  146 + obj.put(item1, sdf.format(new Date(obj.getLong(item1))))
  147 + );
148 148 return obj;
149   - }).collect(Collectors.toList());
  149 + })
  150 + .collect(Collectors.toList())
  151 + ;
150 152 }
151 153  
152 154 }
... ...
src/main/java/cn/fw/freya/service/CrawlBizService.java
  View file @5736d4c
... ... @@ -21,8 +21,10 @@ import java.time.LocalDate;
21 21 import java.util.List;
22 22 import java.util.Map;
23 23 import java.util.Objects;
  24 +import java.util.concurrent.ArrayBlockingQueue;
24 25 import java.util.concurrent.TimeUnit;
25 26 import java.util.concurrent.locks.LockSupport;
  27 +import java.util.function.Function;
26 28 import java.util.stream.Collectors;
27 29  
28 30 /**
... ... @@ -45,12 +47,24 @@ public class CrawlBizService {
45 47 final AccountRpcService accountRpcService,
46 48 final ReportRpcService reportRpcService) {
47 49 this.crawlStrategyMap = crawlStrategyList.stream()
48   - .collect(Collectors.toMap(CrawlStrategy::getType, v -> v));
  50 + .collect(Collectors.toMap(CrawlStrategy::getType, Function.identity()));
49 51 this.accountService = accountService;
50 52 this.accountRpcService = accountRpcService;
51 53 this.reportRpcService = reportRpcService;
52 54 }
53 55  
  56 + public static void main(String[] args) throws InterruptedException {
  57 + ArrayBlockingQueue<Integer> queue = new ArrayBlockingQueue<>(5);
  58 + queue.add(1);
  59 + queue.add(2);
  60 + queue.add(3);
  61 + queue.offer(4);
  62 + queue.offer(5);
  63 + queue.take();
  64 + queue.put(6);
  65 + System.out.println();
  66 + }
  67 +
54 68 /**
55 69 * 登陆准备
56 70 *
... ...
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
  View file @5736d4c
... ... @@ -82,7 +82,7 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
82 82 private final Common common;
83 83 public static final ConcurrentHashMap<String, WebDriver> DRIVER_MAP = new ConcurrentHashMap<>();
84 84 private final AccountService accountService;
85   - private final String playbackBaseUrl = "https://live.kuaishou.com/playback/";
  85 + private static final String playbackBaseUrl = "https://live.kuaishou.com/playback/";
86 86 private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>();
87 87 private final AccountRpcService accountRpcService;
88 88 private static final String RE_TRY_GET_SIG_FLAG = "reTry";
... ... @@ -652,8 +652,8 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
652 652 }
653 653 }
654 654 if (Objects.nonNull(playbackMsg)) {
655   - object.put("coverUrl", playbackMsg.getString("coverUrl"));
656   - object.put("playbackUrl", this.playbackBaseUrl + playbackMsg.getString("productId"));
  655 + object.put("coverUrl", playbackMsg.getString("poster"));
  656 + object.put("playbackUrl", this.playbackBaseUrl + playbackMsg.getString("id"));
657 657 } else {
658 658 object.put("hasInvalidPlayback", true);
659 659 }
... ... @@ -666,16 +666,19 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
666 666 if (Objects.isNull(searchKey))
667 667 return objects;
668 668 Map<String, Object> params = new LinkedHashMap<>();
669   - Map<String, Object> params1 = new LinkedHashMap<>();
  669 + /*Map<String, Object> params1 = new LinkedHashMap<>();
  670 + params1.put("count", 150);
670 671 params1.put("principalId", searchKey);
671 672 params1.put("pcursor", "");
672   - params1.put("count", 150);
673 673 params.put("operationName", "playbackFeedsQuery");
674 674 params.put("variables", params1);
675 675 params.put("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" +
676 676 " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " +
677 677 "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"
678   - );
  678 + );*/
  679 + params.put("count", 150);
  680 + params.put("principalId", searchKey);
  681 + params.put("pcursor", "");
679 682 /*HttpConfig config = HttpConfig.custom()
680 683 .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName())
681 684 .url("https://live.kuaishou.com/live_graphql")
... ... @@ -689,14 +692,14 @@ public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {
689 692 .build()
690 693 );
691 694 String res = RequestUtil.post(config);*/
692   - final String res = RequestUtil.sendPostBody("https://live.kuaishou.com/live_graphql", params);
  695 + final String res = RequestUtil.httpGet("https://live.kuaishou.com/live_api/playback/list", params);
693 696 log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res));
694 697 if (!StringUtils.hasText(res)) {
695 698 return objects;
696 699 }
697 700 JSONObject resObj = JSON.parseObject(res);
698 701 try {
699   - return resObj.getJSONObject("data").getJSONObject("playbackFeeds").getJSONArray("list");
  702 + return resObj.getJSONObject("data").getJSONArray("list");
700 703 } catch (Exception e) {
701 704 log.error(String.format("%s 抓取[%s]平台账户号为: %s的直播回放数据出现异常!!! 异常信息为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, e.getMessage()));
702 705 return objects;
... ...
src/main/java/cn/fw/freya/service/rpc/ReportRpcService.java
  View file @5736d4c
... ... @@ -76,7 +76,7 @@ public class ReportRpcService {
76 76 */
77 77 public boolean reportVideo(String accountNo, Integer type, List<VideoPool> videoList) {
78 78 VideoPool videoPool;
79   - if (videoList.size() > 0) {
  79 + if (!videoList.isEmpty()) {
80 80 videoPool = videoList.get(0);
81 81 } else {
82 82 return true;
... ... @@ -133,7 +133,7 @@ public class ReportRpcService {
133 133 */
134 134 public boolean reportLive(String accountNo, Integer type, List<LivePool> liveList) {
135 135 LivePool livePool;
136   - if (liveList.size() > 0) {
  136 + if (!liveList.isEmpty()) {
137 137 livePool = liveList.get(0);
138 138 } else {
139 139 return true;
... ...
src/main/java/cn/fw/freya/utils/RequestUtil.java
  View file @5736d4c
... ... @@ -2,7 +2,6 @@ package cn.fw.freya.utils;
2 2  
3 3 import cn.fw.freya.common.Callback;
4 4 import cn.fw.freya.utils.http.*;
5   -import com.alibaba.fastjson.JSON;
6 5 import com.alibaba.fastjson.JSONObject;
7 6 import lombok.extern.slf4j.Slf4j;
8 7 import org.apache.http.*;
... ... @@ -737,6 +736,25 @@ public class RequestUtil {
737 736 * 发送HttpGet请求
738 737 *
739 738 * @param getUrl 请求地址
  739 + * @param params 请求参数
  740 + * @return
  741 + */
  742 + public static String httpGet(String getUrl, Map<String, Object> params) {
  743 + PublicUtil.assertTrue(Objects.nonNull(params) && !params.isEmpty(), "参数Map不能为空");
  744 + StringBuilder paramsMsg = new StringBuilder();
  745 + paramsMsg.append("?");
  746 + params.forEach((k, v) -> paramsMsg.append(k)
  747 + .append("=")
  748 + .append(v.toString())
  749 + .append("&")
  750 + );
  751 + return httpGet(getUrl + paramsMsg.substring(0, paramsMsg.length() - 1));
  752 + }
  753 +
  754 + /**
  755 + * 发送HttpGet请求
  756 + *
  757 + * @param getUrl 请求地址
740 758 * @return
741 759 */
742 760 public static String httpGet(String getUrl) {
... ... @@ -912,7 +930,7 @@ public class RequestUtil {
912 930 }
913 931  
914 932 public static void main(String[] args) throws Exception {
915   - List<BasicNameValuePair> params = new ArrayList<>();
  933 + /*List<BasicNameValuePair> params = new ArrayList<>();
916 934 final BasicNameValuePair pair1 = new BasicNameValuePair("principalId", "NXXZX8888");
917 935 final BasicNameValuePair pair2 = new BasicNameValuePair("pcursor", "");
918 936 final BasicNameValuePair pair3 = new BasicNameValuePair("count", "150");
... ... @@ -928,7 +946,30 @@ public class RequestUtil {
928 946 "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}");
929 947 params1.add(pair5);
930 948 params1.add(pair6);
931   - System.out.println(proxyPOST("https://live.kuaishou.com/live_graphql", params1, "27.159.66.41", 20108));
  949 + System.out.println(proxyPOST("https://live.kuaishou.com/live_graphql", params1, "27.159.66.41", 20108));*/
  950 + Map<String, Object> params = new LinkedHashMap<>();
  951 + /*Map<String, Object> params1 = new LinkedHashMap<>();
  952 + params1.put("count", 150);
  953 + params1.put("principalId", searchKey);
  954 + params1.put("pcursor", "");
  955 + params.put("operationName", "playbackFeedsQuery");
  956 + params.put("variables", params1);
  957 + params.put("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" +
  958 + " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " +
  959 + "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"
  960 + );*/
  961 + params.put("count", 150);
  962 + params.put("principalId", "plabsdfk");
  963 + params.put("pcursor", "");
  964 + params.put("hasmore", true);
  965 +
  966 + StringBuilder paramsMsg = new StringBuilder();
  967 + params.forEach((k, v) -> paramsMsg.append(k)
  968 + .append("=")
  969 + .append(v.toString())
  970 + .append("&")
  971 + );
  972 + System.out.println(paramsMsg.substring(0, paramsMsg.length() - 1));
932 973 }
933 974  
934 975 }
... ...