2022年5月27日11:26:26 更换http工具请求快手直播回放数据

王明元
1 parent 16d45828
Showing 3 changed files with 183 additions and 8 deletions
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
src/main/java/cn/fw/freya/task/DataCaptureTask.java
src/main/java/cn/fw/freya/utils/RequestUtil.java
@@ -19,7 +19,6 @@ import cn.fw.freya.utils.RequestUtil;
 import cn.fw.freya.utils.http.HttpConfig;
 import cn.fw.freya.utils.http.HttpCookies;
 import cn.fw.freya.utils.http.HttpHeader;
-import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.RequiredArgsConstructor;
@@ -509,7 +508,7 @@ public class KuaiShouCrawl implements CrawlStrategy {
                 " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " +
                 "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"
         );
-        HttpConfig config = HttpConfig.custom()
+        /*HttpConfig config = HttpConfig.custom()
                 .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName())
                 .url("https://live.kuaishou.com/live_graphql")
                 .json(JSON.toJSONString(params))
@@ -521,7 +520,8 @@ public class KuaiShouCrawl implements CrawlStrategy {
                         .cookie(this.getRandomUserCookies())
                         .build()
                 );
-        String res = RequestUtil.post(config);
+        String res = RequestUtil.post(config);*/
+        final String res = RequestUtil.sendPostBody("https://live.kuaishou.com/live_graphql", params);
         log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res));
         if (!StringUtils.hasText(res)) {
             return objects;
@@ -97,7 +97,7 @@ public class DataCaptureTask {
     /**
      * 每2分钟执行抓取数据
      */
-    @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
+    //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000)
     public void captureLivePlayback() {
         final Random random = new Random();
         List<LivePool> withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d)
@@ -2,6 +2,7 @@ package cn.fw.freya.utils;
  
 import cn.fw.freya.common.Callback;
 import cn.fw.freya.utils.http.*;
+import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
@@ -24,11 +25,9 @@ import org.apache.http.util.EntityUtils;
 import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
  
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
+import java.io.*;
 import java.net.*;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.stream.Collectors;
  
@@ -671,4 +670,180 @@ public class RequestUtil {
             return new PasswordAuthentication(user, password.toCharArray());
         }
     }
+
+    /**
+     * 发送Post请求，参数拼接到url
+     *
+     * @param httpUrl 请求路径
+     * @param map 请求参数
+     * @return
+     */
+    public static String sendPost(String httpUrl, Map<String, String> map) {
+        HttpURLConnection connection = null;
+        InputStream is = null;
+        OutputStream os = null;
+        BufferedReader br = null;
+        String result = null;
+        try {
+            StringBuilder param = new StringBuilder();
+            boolean isFirstPar = true;
+            for (String key : map.keySet()) {
+                if (!isFirstPar) {
+                    param.append("&");
+                }
+                isFirstPar = false;
+                String val = URLEncoder.encode(map.get(key), StandardCharsets.UTF_8);
+                param.append(String.format("%1$s=%2$s", key, val));
+            }
+            URL url = new URL(httpUrl);
+            connection = (HttpURLConnection) url.openConnection();// 通过远程url连接对象打开连接
+            connection.setRequestMethod("POST");// 设置连接请求方式
+            connection.setConnectTimeout(3000);// 设置连接主机服务器超时时间
+            connection.setReadTimeout(3000);// 设置读取主机服务器返回数据超时时间
+            connection.setDoOutput(true);// 默认值为：false，当向远程服务器传送数据/写数据时，需要设置为true
+            connection.setDoInput(true);// 默认值为：true，当前向远程服务读取数据时，设置为true，该参数可有可无
+            connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");// 设置传入参数的格式: 请求参数应该是 name1=value1&name2=value2 的形式。
+            os = connection.getOutputStream();// 通过连接对象获取一个输出流
+            os.write(param.toString().getBytes());// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的
+            if (connection.getResponseCode() == 200) {// 通过连接对象获取一个输入流，向远程读取
+                is = connection.getInputStream();
+                // 对输入流对象进行包装:charset根据工作项目组的要求来设置
+                br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
+                StringBuilder sb = new StringBuilder();
+                String temp;
+                // 循环遍历一行一行读取数据
+                while ((temp = br.readLine()) != null) {
+                    sb.append(temp);
+                    sb.append("\r\n");
+                }
+                result = sb.toString();
+            }
+        } catch (IOException e) {
+            log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage()));
+        } finally {// 使用finally块释放资源
+            try {
+                if (Objects.nonNull(br)) {
+                    br.close();
+                }
+                if (Objects.nonNull(os)) {
+                    os.close();
+                }
+                if (Objects.nonNull(is)) {
+                    is.close();
+                }
+            } catch (IOException e) {
+                log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage()));
+            }
+            if (Objects.nonNull(connection)) {// 断开与远程地址url的连接
+                connection.disconnect();
+            }
+        }
+        return result;
+    }
+
+    /**
+     * 发送Post请求，参数放到Body中
+     *
+     * @param url 请求路径
+     * @param map 参数体
+     * @return 返回结果
+     */
+    public static String sendPostBody(String url, Map<String, Object> map) {
+        OutputStreamWriter out = null;
+        BufferedReader in = null;
+        StringBuilder result = new StringBuilder();
+        try {
+            URL realUrl = new URL(url);
+            URLConnection conn = realUrl.openConnection();// 打开和URL之间的连接
+            // 设置通用的请求属性
+            conn.setRequestProperty("accept", "*/*");
+            conn.setRequestProperty("connection", "Keep-Alive");
+            conn.setRequestProperty("Content-Type", "application/json");
+            // 发送POST请求必须设置如下两行
+            conn.setDoOutput(true);
+            conn.setDoInput(true);
+            out = new OutputStreamWriter(conn.getOutputStream(), StandardCharsets.UTF_8);// 获取URLConnection对象对应的输出流
+            // 发送请求参数
+            if (Objects.nonNull(map)) {
+                JSONObject obj = new JSONObject();
+                for (Map.Entry<String, Object> mt : map.entrySet()) {
+                    obj.put(mt.getKey(), mt.getValue());
+                }
+                out.write(obj.toJSONString());
+            }
+            out.flush();// flush输出流的缓冲
+            in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));// 定义BufferedReader输入流来读取URL的响应
+            String line;
+            while ((line = in.readLine()) != null) {
+                result.append(line);
+            }
+        } catch (Exception e) {
+            log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage()));
+            return null;
+        } finally {// 使用finally块释放资源
+            try {
+                if (Objects.nonNull(in)) {
+                    in.close();
+                }
+                if (Objects.nonNull(out)) {
+                    out.close();
+                }
+            } catch (IOException e) {
+                log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage()));
+            }
+        }
+        return result.toString();
+    }
+
+    /**
+     * 发送HttpGet请求
+     *
+     * @param getUrl 请求地址
+     * @return
+     */
+    public static String httpGet(String getUrl) {
+        StringBuilder buffer = new StringBuilder();
+        HttpURLConnection connection = null;
+        InputStream inputStream = null;
+        InputStreamReader inputStreamReader = null;
+        BufferedReader bufferedReader = null;
+        try {
+            URL url = new URL(getUrl);
+            connection = (HttpURLConnection) url.openConnection();
+            connection.setDoOutput(false);
+            connection.setDoInput(true);
+            connection.setUseCaches(false);
+            connection.setRequestMethod("GET");
+            connection.connect();
+            // 将返回的输入流转换成字符串
+            inputStream = connection.getInputStream();
+            inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+            bufferedReader = new BufferedReader(inputStreamReader);
+            String str;
+            while ((str = bufferedReader.readLine()) != null) {
+                buffer.append(str);
+            }
+        } catch (Exception e) {
+            log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage()));
+        } finally {// 使用finally块释放资源
+            try {
+                if (Objects.nonNull(bufferedReader)) {
+                    bufferedReader.close();
+                }
+                if (Objects.nonNull(inputStreamReader)) {
+                    inputStreamReader.close();
+                }
+                if (Objects.nonNull(inputStream)) {
+                    inputStream.close();
+                }
+            } catch (Exception e) {
+                log.error(String.format("释放资源发生异常, 异常信息为: %s", e.getMessage()));
+            }
+            if (Objects.nonNull(connection)) {
+                connection.disconnect();
+            }
+        }
+        return buffer.toString();
+    }
+
 }