diff --git a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java index cc4cf55..592becf 100644 --- a/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java +++ b/src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java @@ -19,7 +19,6 @@ import cn.fw.freya.utils.RequestUtil; import cn.fw.freya.utils.http.HttpConfig; import cn.fw.freya.utils.http.HttpCookies; import cn.fw.freya.utils.http.HttpHeader; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import lombok.RequiredArgsConstructor; @@ -509,7 +508,7 @@ public class KuaiShouCrawl implements CrawlStrategy { " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " + "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}" ); - HttpConfig config = HttpConfig.custom() + /*HttpConfig config = HttpConfig.custom() .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName()) .url("https://live.kuaishou.com/live_graphql") .json(JSON.toJSONString(params)) @@ -521,7 +520,8 @@ public class KuaiShouCrawl implements CrawlStrategy { .cookie(this.getRandomUserCookies()) .build() ); - String res = RequestUtil.post(config); + String res = RequestUtil.post(config);*/ + final String res = RequestUtil.sendPostBody("https://live.kuaishou.com/live_graphql", params); log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res)); if (!StringUtils.hasText(res)) { return objects; diff --git a/src/main/java/cn/fw/freya/task/DataCaptureTask.java b/src/main/java/cn/fw/freya/task/DataCaptureTask.java index 28de263..a0eb28b 100644 --- a/src/main/java/cn/fw/freya/task/DataCaptureTask.java +++ b/src/main/java/cn/fw/freya/task/DataCaptureTask.java @@ -97,7 +97,7 @@ public class DataCaptureTask { /** * 每2分钟执行抓取数据 */ - @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) + //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) public void captureLivePlayback() { final Random random = new Random(); List withoutPlaybackLive = common.getWithoutPlaybackLive(1, 60d) diff --git a/src/main/java/cn/fw/freya/utils/RequestUtil.java b/src/main/java/cn/fw/freya/utils/RequestUtil.java index d55b406..ba18804 100644 --- a/src/main/java/cn/fw/freya/utils/RequestUtil.java +++ b/src/main/java/cn/fw/freya/utils/RequestUtil.java @@ -2,6 +2,7 @@ package cn.fw.freya.utils; import cn.fw.freya.common.Callback; import cn.fw.freya.utils.http.*; +import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -24,11 +25,9 @@ import org.apache.http.util.EntityUtils; import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; +import java.io.*; import java.net.*; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; @@ -671,4 +670,180 @@ public class RequestUtil { return new PasswordAuthentication(user, password.toCharArray()); } } + + /** + * 发送Post请求,参数拼接到url + * + * @param httpUrl 请求路径 + * @param map 请求参数 + * @return + */ + public static String sendPost(String httpUrl, Map map) { + HttpURLConnection connection = null; + InputStream is = null; + OutputStream os = null; + BufferedReader br = null; + String result = null; + try { + StringBuilder param = new StringBuilder(); + boolean isFirstPar = true; + for (String key : map.keySet()) { + if (!isFirstPar) { + param.append("&"); + } + isFirstPar = false; + String val = URLEncoder.encode(map.get(key), StandardCharsets.UTF_8); + param.append(String.format("%1$s=%2$s", key, val)); + } + URL url = new URL(httpUrl); + connection = (HttpURLConnection) url.openConnection();// 通过远程url连接对象打开连接 + connection.setRequestMethod("POST");// 设置连接请求方式 + connection.setConnectTimeout(3000);// 设置连接主机服务器超时时间 + connection.setReadTimeout(3000);// 设置读取主机服务器返回数据超时时间 + connection.setDoOutput(true);// 默认值为:false,当向远程服务器传送数据/写数据时,需要设置为true + connection.setDoInput(true);// 默认值为:true,当前向远程服务读取数据时,设置为true,该参数可有可无 + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");// 设置传入参数的格式: 请求参数应该是 name1=value1&name2=value2 的形式。 + os = connection.getOutputStream();// 通过连接对象获取一个输出流 + os.write(param.toString().getBytes());// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的 + if (connection.getResponseCode() == 200) {// 通过连接对象获取一个输入流,向远程读取 + is = connection.getInputStream(); + // 对输入流对象进行包装:charset根据工作项目组的要求来设置 + br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + String temp; + // 循环遍历一行一行读取数据 + while ((temp = br.readLine()) != null) { + sb.append(temp); + sb.append("\r\n"); + } + result = sb.toString(); + } + } catch (IOException e) { + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); + } finally {// 使用finally块释放资源 + try { + if (Objects.nonNull(br)) { + br.close(); + } + if (Objects.nonNull(os)) { + os.close(); + } + if (Objects.nonNull(is)) { + is.close(); + } + } catch (IOException e) { + log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage())); + } + if (Objects.nonNull(connection)) {// 断开与远程地址url的连接 + connection.disconnect(); + } + } + return result; + } + + /** + * 发送Post请求,参数放到Body中 + * + * @param url 请求路径 + * @param map 参数体 + * @return 返回结果 + */ + public static String sendPostBody(String url, Map map) { + OutputStreamWriter out = null; + BufferedReader in = null; + StringBuilder result = new StringBuilder(); + try { + URL realUrl = new URL(url); + URLConnection conn = realUrl.openConnection();// 打开和URL之间的连接 + // 设置通用的请求属性 + conn.setRequestProperty("accept", "*/*"); + conn.setRequestProperty("connection", "Keep-Alive"); + conn.setRequestProperty("Content-Type", "application/json"); + // 发送POST请求必须设置如下两行 + conn.setDoOutput(true); + conn.setDoInput(true); + out = new OutputStreamWriter(conn.getOutputStream(), StandardCharsets.UTF_8);// 获取URLConnection对象对应的输出流 + // 发送请求参数 + if (Objects.nonNull(map)) { + JSONObject obj = new JSONObject(); + for (Map.Entry mt : map.entrySet()) { + obj.put(mt.getKey(), mt.getValue()); + } + out.write(obj.toJSONString()); + } + out.flush();// flush输出流的缓冲 + in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));// 定义BufferedReader输入流来读取URL的响应 + String line; + while ((line = in.readLine()) != null) { + result.append(line); + } + } catch (Exception e) { + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); + return null; + } finally {// 使用finally块释放资源 + try { + if (Objects.nonNull(in)) { + in.close(); + } + if (Objects.nonNull(out)) { + out.close(); + } + } catch (IOException e) { + log.error(String.format("关闭流发生异常, 异常信息为: %s", e.getMessage())); + } + } + return result.toString(); + } + + /** + * 发送HttpGet请求 + * + * @param getUrl 请求地址 + * @return + */ + public static String httpGet(String getUrl) { + StringBuilder buffer = new StringBuilder(); + HttpURLConnection connection = null; + InputStream inputStream = null; + InputStreamReader inputStreamReader = null; + BufferedReader bufferedReader = null; + try { + URL url = new URL(getUrl); + connection = (HttpURLConnection) url.openConnection(); + connection.setDoOutput(false); + connection.setDoInput(true); + connection.setUseCaches(false); + connection.setRequestMethod("GET"); + connection.connect(); + // 将返回的输入流转换成字符串 + inputStream = connection.getInputStream(); + inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + bufferedReader = new BufferedReader(inputStreamReader); + String str; + while ((str = bufferedReader.readLine()) != null) { + buffer.append(str); + } + } catch (Exception e) { + log.error(String.format("发送请求发生异常, 异常信息为: %s", e.getMessage())); + } finally {// 使用finally块释放资源 + try { + if (Objects.nonNull(bufferedReader)) { + bufferedReader.close(); + } + if (Objects.nonNull(inputStreamReader)) { + inputStreamReader.close(); + } + if (Objects.nonNull(inputStream)) { + inputStream.close(); + } + } catch (Exception e) { + log.error(String.format("释放资源发生异常, 异常信息为: %s", e.getMessage())); + } + if (Objects.nonNull(connection)) { + connection.disconnect(); + } + } + return buffer.toString(); + } + }