Commit 90e8be74a4038d6710873eeb3964e77cc9543d64
1 parent
16d45828
2022年5月27日11:26:26 更换http工具请求快手直播回放数据
Showing
3 changed files
with
183 additions
and
8 deletions
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
@@ -19,7 +19,6 @@ import cn.fw.freya.utils.RequestUtil; | @@ -19,7 +19,6 @@ import cn.fw.freya.utils.RequestUtil; | ||
19 | import cn.fw.freya.utils.http.HttpConfig; | 19 | import cn.fw.freya.utils.http.HttpConfig; |
20 | import cn.fw.freya.utils.http.HttpCookies; | 20 | import cn.fw.freya.utils.http.HttpCookies; |
21 | import cn.fw.freya.utils.http.HttpHeader; | 21 | import cn.fw.freya.utils.http.HttpHeader; |
22 | -import com.alibaba.fastjson.JSON; | ||
23 | import com.alibaba.fastjson.JSONArray; | 22 | import com.alibaba.fastjson.JSONArray; |
24 | import com.alibaba.fastjson.JSONObject; | 23 | import com.alibaba.fastjson.JSONObject; |
25 | import lombok.RequiredArgsConstructor; | 24 | import lombok.RequiredArgsConstructor; |
@@ -509,7 +508,7 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -509,7 +508,7 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
509 | " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " + | 508 | " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " + |
510 | "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}" | 509 | "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}" |
511 | ); | 510 | ); |
512 | - HttpConfig config = HttpConfig.custom() | 511 | + /*HttpConfig config = HttpConfig.custom() |
513 | .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName()) | 512 | .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName()) |
514 | .url("https://live.kuaishou.com/live_graphql") | 513 | .url("https://live.kuaishou.com/live_graphql") |
515 | .json(JSON.toJSONString(params)) | 514 | .json(JSON.toJSONString(params)) |
@@ -521,7 +520,8 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -521,7 +520,8 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
521 | .cookie(this.getRandomUserCookies()) | 520 | .cookie(this.getRandomUserCookies()) |
522 | .build() | 521 | .build() |
523 | ); | 522 | ); |
524 | - String res = RequestUtil.post(config); | 523 | + String res = RequestUtil.post(config);*/ |
524 | + final String res = RequestUtil.sendPostBody("https://live.kuaishou.com/live_graphql", params); | ||
525 | log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res)); | 525 | log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res)); |
526 | if (!StringUtils.hasText(res)) { | 526 | if (!StringUtils.hasText(res)) { |
527 | return objects; | 527 | return objects; |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
@@ -97,7 +97,7 @@ public class DataCaptureTask { | @@ -97,7 +97,7 @@ public class DataCaptureTask { | ||
97 | /** | 97 | /** |
98 | * 每2分钟执行抓取数据 | 98 | * 每2分钟执行抓取数据 |
99 | */ | 99 | */ |
100 | - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) | 100 | + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) |
101 | public void captureLivePlayback() { | 101 | public void captureLivePlayback() { |
102 | final Random random = new Random(); | 102 | final Random random = new Random(); |
103 | List<LivePool> withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) | 103 | List<LivePool> withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) |
src/main/java/cn/fw/freya/utils/RequestUtil.java
@@ -2,6 +2,7 @@ package cn.fw.freya.utils; | @@ -2,6 +2,7 @@ package cn.fw.freya.utils; | ||
2 | 2 | ||
3 | import cn.fw.freya.common.Callback; | 3 | import cn.fw.freya.common.Callback; |
4 | import cn.fw.freya.utils.http.*; | 4 | import cn.fw.freya.utils.http.*; |
5 | +import com.alibaba.fastjson.JSONObject; | ||
5 | import lombok.extern.slf4j.Slf4j; | 6 | import lombok.extern.slf4j.Slf4j; |
6 | import org.apache.http.Header; | 7 | import org.apache.http.Header; |
7 | import org.apache.http.HttpEntity; | 8 | import org.apache.http.HttpEntity; |
@@ -24,11 +25,9 @@ import org.apache.http.util.EntityUtils; | @@ -24,11 +25,9 @@ import org.apache.http.util.EntityUtils; | ||
24 | import org.springframework.lang.NonNull; | 25 | import org.springframework.lang.NonNull; |
25 | import org.springframework.lang.Nullable; | 26 | import org.springframework.lang.Nullable; |
26 | 27 | ||
27 | -import java.io.BufferedReader; | ||
28 | -import java.io.IOException; | ||
29 | -import java.io.InputStreamReader; | ||
30 | -import java.io.OutputStream; | 28 | +import java.io.*; |
31 | import java.net.*; | 29 | import java.net.*; |
30 | +import java.nio.charset.StandardCharsets; | ||
32 | import java.util.*; | 31 | import java.util.*; |
33 | import java.util.stream.Collectors; | 32 | import java.util.stream.Collectors; |
34 | 33 | ||
@@ -671,4 +670,180 @@ public class RequestUtil { | @@ -671,4 +670,180 @@ public class RequestUtil { | ||
671 | return new PasswordAuthentication(user, password.toCharArray()); | 670 | return new PasswordAuthentication(user, password.toCharArray()); |
672 | } | 671 | } |
673 | } | 672 | } |
673 | + | ||
674 | + /** | ||
675 | + * 发送Post请求,参数拼接到url | ||
676 | + * | ||
677 | + * @param httpUrl 请求路径 | ||
678 | + * @param map 请求参数 | ||
679 | + * @return | ||
680 | + */ | ||
681 | + public static String sendPost(String httpUrl, Map<String, String> map) { | ||
682 | + HttpURLConnection connection = null; | ||
683 | + InputStream is = null; | ||
684 | + OutputStream os = null; | ||
685 | + BufferedReader br = null; | ||
686 | + String result = null; | ||
687 | + try { | ||
688 | + StringBuilder param = new StringBuilder(); | ||
689 | + boolean isFirstPar = true; | ||
690 | + for (String key : map.keySet()) { | ||
691 | + if (!isFirstPar) { | ||
692 | + param.append("&"); | ||
693 | + } | ||
694 | + isFirstPar = false; | ||
695 | + String val = URLEncoder.encode(map.get(key), StandardCharsets.UTF_8); | ||
696 | + param.append(String.format("%1$s=%2$s", key, val)); | ||
697 | + } | ||
698 | + URL url = new URL(httpUrl); | ||
699 | + connection = (HttpURLConnection) url.openConnection();// 通过远程url连接对象打开连接 | ||
700 | + connection.setRequestMethod("POST");// 设置连接请求方式 | ||
701 | + connection.setConnectTimeout(3000);// 设置连接主机服务器超时时间 | ||
702 | + connection.setReadTimeout(3000);// 设置读取主机服务器返回数据超时时间 | ||
703 | + connection.setDoOutput(true);// 默认值为:false,当向远程服务器传送数据/写数据时,需要设置为true | ||
704 | + connection.setDoInput(true);// 默认值为:true,当前向远程服务读取数据时,设置为true,该参数可有可无 | ||
705 | + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");// 设置传入参数的格式: 请求参数应该是 name1=value1&name2=value2 的形式。 | ||
706 | + os = connection.getOutputStream();// 通过连接对象获取一个输出流 | ||
707 | + os.write(param.toString().getBytes());// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的 | ||
708 | + if (connection.getResponseCode() == 200) {// 通过连接对象获取一个输入流,向远程读取 | ||
709 | + is = connection.getInputStream(); | ||
710 | + // 对输入流对象进行包装:charset根据工作项目组的要求来设置 | ||
711 | + br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); | ||
712 | + StringBuilder sb = new StringBuilder(); | ||
713 | + String temp; | ||
714 | + // 循环遍历一行一行读取数据 | ||
715 | + while ((temp = br.readLine()) != null) { | ||
716 | + sb.append(temp); | ||
717 | + sb.append("\r\n"); | ||
718 | + } | ||
719 | + result = sb.toString(); | ||
720 | + } | ||
721 | + } catch (IOException e) { | ||
722 | + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); | ||
723 | + } finally {// 使用finally块释放资源 | ||
724 | + try { | ||
725 | + if (Objects.nonNull(br)) { | ||
726 | + br.close(); | ||
727 | + } | ||
728 | + if (Objects.nonNull(os)) { | ||
729 | + os.close(); | ||
730 | + } | ||
731 | + if (Objects.nonNull(is)) { | ||
732 | + is.close(); | ||
733 | + } | ||
734 | + } catch (IOException e) { | ||
735 | + log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage())); | ||
736 | + } | ||
737 | + if (Objects.nonNull(connection)) {// 断开与远程地址url的连接 | ||
738 | + connection.disconnect(); | ||
739 | + } | ||
740 | + } | ||
741 | + return result; | ||
742 | + } | ||
743 | + | ||
744 | + /** | ||
745 | + * 发送Post请求,参数放到Body中 | ||
746 | + * | ||
747 | + * @param url 请求路径 | ||
748 | + * @param map 参数体 | ||
749 | + * @return 返回结果 | ||
750 | + */ | ||
751 | + public static String sendPostBody(String url, Map<String, Object> map) { | ||
752 | + OutputStreamWriter out = null; | ||
753 | + BufferedReader in = null; | ||
754 | + StringBuilder result = new StringBuilder(); | ||
755 | + try { | ||
756 | + URL realUrl = new URL(url); | ||
757 | + URLConnection conn = realUrl.openConnection();// 打开和URL之间的连接 | ||
758 | + // 设置通用的请求属性 | ||
759 | + conn.setRequestProperty("accept", "*/*"); | ||
760 | + conn.setRequestProperty("connection", "Keep-Alive"); | ||
761 | + conn.setRequestProperty("Content-Type", "application/json"); | ||
762 | + // 发送POST请求必须设置如下两行 | ||
763 | + conn.setDoOutput(true); | ||
764 | + conn.setDoInput(true); | ||
765 | + out = new OutputStreamWriter(conn.getOutputStream(), StandardCharsets.UTF_8);// 获取URLConnection对象对应的输出流 | ||
766 | + // 发送请求参数 | ||
767 | + if (Objects.nonNull(map)) { | ||
768 | + JSONObject obj = new JSONObject(); | ||
769 | + for (Map.Entry<String, Object> mt : map.entrySet()) { | ||
770 | + obj.put(mt.getKey(), mt.getValue()); | ||
771 | + } | ||
772 | + out.write(obj.toJSONString()); | ||
773 | + } | ||
774 | + out.flush();// flush输出流的缓冲 | ||
775 | + in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));// 定义BufferedReader输入流来读取URL的响应 | ||
776 | + String line; | ||
777 | + while ((line = in.readLine()) != null) { | ||
778 | + result.append(line); | ||
779 | + } | ||
780 | + } catch (Exception e) { | ||
781 | + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); | ||
782 | + return null; | ||
783 | + } finally {// 使用finally块释放资源 | ||
784 | + try { | ||
785 | + if (Objects.nonNull(in)) { | ||
786 | + in.close(); | ||
787 | + } | ||
788 | + if (Objects.nonNull(out)) { | ||
789 | + out.close(); | ||
790 | + } | ||
791 | + } catch (IOException e) { | ||
792 | + log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage())); | ||
793 | + } | ||
794 | + } | ||
795 | + return result.toString(); | ||
796 | + } | ||
797 | + | ||
798 | + /** | ||
799 | + * 发送HttpGet请求 | ||
800 | + * | ||
801 | + * @param getUrl 请求地址 | ||
802 | + * @return | ||
803 | + */ | ||
804 | + public static String httpGet(String getUrl) { | ||
805 | + StringBuilder buffer = new StringBuilder(); | ||
806 | + HttpURLConnection connection = null; | ||
807 | + InputStream inputStream = null; | ||
808 | + InputStreamReader inputStreamReader = null; | ||
809 | + BufferedReader bufferedReader = null; | ||
810 | + try { | ||
811 | + URL url = new URL(getUrl); | ||
812 | + connection = (HttpURLConnection) url.openConnection(); | ||
813 | + connection.setDoOutput(false); | ||
814 | + connection.setDoInput(true); | ||
815 | + connection.setUseCaches(false); | ||
816 | + connection.setRequestMethod("GET"); | ||
817 | + connection.connect(); | ||
818 | + // 将返回的输入流转换成字符串 | ||
819 | + inputStream = connection.getInputStream(); | ||
820 | + inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); | ||
821 | + bufferedReader = new BufferedReader(inputStreamReader); | ||
822 | + String str; | ||
823 | + while ((str = bufferedReader.readLine()) != null) { | ||
824 | + buffer.append(str); | ||
825 | + } | ||
826 | + } catch (Exception e) { | ||
827 | + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); | ||
828 | + } finally {// 使用finally块释放资源 | ||
829 | + try { | ||
830 | + if (Objects.nonNull(bufferedReader)) { | ||
831 | + bufferedReader.close(); | ||
832 | + } | ||
833 | + if (Objects.nonNull(inputStreamReader)) { | ||
834 | + inputStreamReader.close(); | ||
835 | + } | ||
836 | + if (Objects.nonNull(inputStream)) { | ||
837 | + inputStream.close(); | ||
838 | + } | ||
839 | + } catch (Exception e) { | ||
840 | + log.error(String.format("释放资源发生异常, 异常信息为: %s", e.getMessage())); | ||
841 | + } | ||
842 | + if (Objects.nonNull(connection)) { | ||
843 | + connection.disconnect(); | ||
844 | + } | ||
845 | + } | ||
846 | + return buffer.toString(); | ||
847 | + } | ||
848 | + | ||
674 | } | 849 | } |