diff --git a/src/main/java/cn/fw/freya/utils/RequestUtil.java b/src/main/java/cn/fw/freya/utils/RequestUtil.java index ba18804..c9133d7 100644 --- a/src/main/java/cn/fw/freya/utils/RequestUtil.java +++ b/src/main/java/cn/fw/freya/utils/RequestUtil.java @@ -2,13 +2,14 @@ package cn.fw.freya.utils; import cn.fw.freya.common.Callback; import cn.fw.freya.utils.http.*; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; +import org.apache.http.*; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.*; import org.apache.http.client.utils.DateUtils; import org.apache.http.config.Lookup; @@ -16,10 +17,11 @@ import org.apache.http.cookie.Cookie; import org.apache.http.cookie.CookieOrigin; import org.apache.http.cookie.CookieSpecProvider; import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.impl.client.CookieSpecRegistries; +import org.apache.http.impl.client.*; import org.apache.http.impl.cookie.BasicClientCookie; import org.apache.http.impl.cookie.DefaultCookieSpec; import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import org.springframework.lang.NonNull; @@ -608,74 +610,10 @@ public class RequestUtil { } /** - * 设置全局代理(更改系统属性方式) - * - * @throws IOException - */ - private void setGlobalProxy() throws IOException { - Properties prop = System.getProperties(); - // 设置http访问要使用的代理服务器的地址 - prop.setProperty("http.proxyHost", "183.45.78.31"); - // 设置http访问要使用的代理服务器的端口 - prop.setProperty("http.proxyPort", "8080"); - // 设置不需要通过代理服务器访问的主机，可以使用*通配符，多个地址用|分隔 - prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); - // 设置安全访问使用的代理服务器地址与端口 - // 它没有https.nonProxyHosts属性，它按照http.nonProxyHosts 中设置的规则访问 - prop.setProperty("https.proxyHost", "183.45.78.31"); - prop.setProperty("https.proxyPort", "443"); - // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机 - prop.setProperty("ftp.proxyHost", "183.45.78.31"); - prop.setProperty("ftp.proxyPort", "21"); - prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); - // socks代理服务器的地址与端口 - prop.setProperty("socksProxyHost", "183.45.78.31"); - prop.setProperty("socksProxyPort", "1080"); - // 设置登陆到代理服务器的用户名和密码 - Authenticator.setDefault(new MyAuthenticator("userName", "Password")); - } - - private static void setProxy() throws IOException { - URL url = new URL("http://www.baidu.com"); - // 创建代理服务器 - InetSocketAddress addr = new InetSocketAddress("202.116.64.93", 3969); - // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理 - Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理 - //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码 - HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问 - InputStreamReader in = new InputStreamReader(connection.getInputStream()); - BufferedReader reader = new BufferedReader(in); - while (true) { - String s = reader.readLine(); - if (s != null) { - System.out.println(s); - } - } - } - - public static void main(String[] args) throws IOException { - setProxy(); - } - - static class MyAuthenticator extends Authenticator { - private String user = ""; - private String password = ""; - - public MyAuthenticator(String user, String password) { - this.user = user; - this.password = password; - } - - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(user, password.toCharArray()); - } - } - - /** * 发送Post请求，参数拼接到url * * @param httpUrl 请求路径 - * @param map 请求参数 + * @param map 请求参数 * @return */ public static String sendPost(String httpUrl, Map map) { @@ -846,4 +784,151 @@ public class RequestUtil { return buffer.toString(); } + /** + * 利用代理服务器请求数据 + * + * @param apiURL 需要请求的接口地址 + * @param params 请求接口传参 + * @param proxyHost 代理服务器路径 + * @param proxyPort 代理服务器端口号 + * @return 返回结果 + * @throws Exception + */ + private static String proxyPOST(String apiURL, List params, String proxyHost, Integer proxyPort) throws Exception { + String result; + HttpPost post = new HttpPost(apiURL); + CloseableHttpClient client = null; + CloseableHttpResponse response = null; + HttpEntity entity = null; + try { + post.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); + + post.setHeader("X-Requested-With", "XMLHttpRequest"); + post.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); + // 表单请求参数 + post.setEntity(new UrlEncodedFormEntity(params)); + // 设置客户端超时时间 + RequestConfig config = RequestConfig.custom() + .setSocketTimeout(50000) + .setConnectTimeout(50000) + .setConnectionRequestTimeout(50000) + .build(); + HttpClientBuilder httpClientBuilder = HttpClients.custom() + .setDefaultRequestConfig(config); + log.info(proxyHost); + // 取代理服务器的ip和端口 + HttpHost targetHost = new HttpHost(proxyHost, proxyPort); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + /*credsProvider.setCredentials( + new AuthScope(targetHost.getHostName(), targetHost.getPort()), + new UsernamePasswordCredentials("", "") + );*/ + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + httpClientBuilder.setProxy(targetHost); + log.info("设置代理完成..."); + client = httpClientBuilder.build(); + log.info("获取客户端请求..."); + // 执行POST请求 + response = client.execute(post); + entity = response.getEntity(); + int statusCode = response.getStatusLine().getStatusCode(); + log.info("执行请求结束...响应状态码:" + statusCode); + result = EntityUtils.toString(entity, "UTF-8"); + } catch (Exception e) { + post.abort(); + throw e; + } finally { + EntityUtils.consume(entity); + // 关闭连接 释放资源 + post.releaseConnection(); + if (response != null) { + response.close(); + } + if (client != null) { + client.close(); + } + } + return result; + } + + /** + * 设置全局代理(更改系统属性方式) + * + * @throws IOException + */ + private void setGlobalProxy() throws IOException { + Properties prop = System.getProperties(); + // 设置http访问要使用的代理服务器的地址 + prop.setProperty("http.proxyHost", "183.45.78.31"); + // 设置http访问要使用的代理服务器的端口 + prop.setProperty("http.proxyPort", "8080"); + // 设置不需要通过代理服务器访问的主机，可以使用*通配符，多个地址用|分隔 + prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); + // 设置安全访问使用的代理服务器地址与端口 + // 它没有https.nonProxyHosts属性，它按照http.nonProxyHosts 中设置的规则访问 + prop.setProperty("https.proxyHost", "183.45.78.31"); + prop.setProperty("https.proxyPort", "443"); + // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机 + prop.setProperty("ftp.proxyHost", "183.45.78.31"); + prop.setProperty("ftp.proxyPort", "21"); + prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); + // socks代理服务器的地址与端口 + prop.setProperty("socksProxyHost", "183.45.78.31"); + prop.setProperty("socksProxyPort", "1080"); + // 设置登陆到代理服务器的用户名和密码 + Authenticator.setDefault(new MyAuthenticator("userName", "Password")); + } + + private static void setProxy() throws IOException { + URL url = new URL("http://www.baidu.com"); + // 创建代理服务器 + InetSocketAddress addr = new InetSocketAddress("27.159.66.41", 20263); + // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理 + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理 + //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码 + HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问 + InputStreamReader in = new InputStreamReader(connection.getInputStream()); + BufferedReader reader = new BufferedReader(in); + while (true) { + String s = reader.readLine(); + if (s != null) { + System.out.println(s); + } + } + } + + static class MyAuthenticator extends Authenticator { + private String user = ""; + private String password = ""; + + public MyAuthenticator(String user, String password) { + this.user = user; + this.password = password; + } + + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(user, password.toCharArray()); + } + } + + public static void main(String[] args) throws Exception { + List params = new ArrayList<>(); + final BasicNameValuePair pair1 = new BasicNameValuePair("principalId", "NXXZX8888"); + final BasicNameValuePair pair2 = new BasicNameValuePair("pcursor", ""); + final BasicNameValuePair pair3 = new BasicNameValuePair("count", "150"); + final BasicNameValuePair pair4 = new BasicNameValuePair("operationName", "playbackFeedsQuery"); + params.add(pair1); + params.add(pair2); + params.add(pair3); + params.add(pair4); + final BasicNameValuePair pair5 = new BasicNameValuePair("variables", JSON.toJSONString(params)); + List params1 = new ArrayList<>(); + final BasicNameValuePair pair6 = new BasicNameValuePair("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" + + " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " + + "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"); + params1.add(pair5); + params1.add(pair6); + System.out.println(proxyPOST("https://live.kuaishou.com/live_graphql", params1, "27.159.66.41", 20108)); + } + } diff --git a/src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java b/src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java index fb584ce..b9759c2 100644 --- a/src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java +++ b/src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java @@ -29,11 +29,11 @@ public class ThreadPoolUtil { synchronized (ThreadPoolUtil.class) {// threadPool == null, 加锁 if (threadPool == null) { threadPool = new ThreadPoolExecutor( + 6, 12, - 24, 60, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(124), + new LinkedBlockingQueue<>(256), new ThreadFactoryBuilder().setNamePrefix("capture-pool-").build(), new ThreadPoolExecutor.DiscardPolicy());// 自行创建线程池, 并将创建好的线程池对象赋值给类的静态成员变量threadPool }