Commit b8357e80d0ccd6068b3a847fae2a7d66b0687ccc

Authored by 王明元
1 parent 28d3e509

2022年5月30日18:20:09 调整线程池参数, 减小内存炸鸡概率

src/main/java/cn/fw/freya/utils/RequestUtil.java
... ... @@ -2,13 +2,14 @@ package cn.fw.freya.utils;
2 2  
3 3 import cn.fw.freya.common.Callback;
4 4 import cn.fw.freya.utils.http.*;
  5 +import com.alibaba.fastjson.JSON;
5 6 import com.alibaba.fastjson.JSONObject;
6 7 import lombok.extern.slf4j.Slf4j;
7   -import org.apache.http.Header;
8   -import org.apache.http.HttpEntity;
9   -import org.apache.http.HttpResponse;
10   -import org.apache.http.NameValuePair;
  8 +import org.apache.http.*;
  9 +import org.apache.http.client.CredentialsProvider;
11 10 import org.apache.http.client.HttpClient;
  11 +import org.apache.http.client.config.RequestConfig;
  12 +import org.apache.http.client.entity.UrlEncodedFormEntity;
12 13 import org.apache.http.client.methods.*;
13 14 import org.apache.http.client.utils.DateUtils;
14 15 import org.apache.http.config.Lookup;
... ... @@ -16,10 +17,11 @@ import org.apache.http.cookie.Cookie;
16 17 import org.apache.http.cookie.CookieOrigin;
17 18 import org.apache.http.cookie.CookieSpecProvider;
18 19 import org.apache.http.cookie.MalformedCookieException;
19   -import org.apache.http.impl.client.CookieSpecRegistries;
  20 +import org.apache.http.impl.client.*;
20 21 import org.apache.http.impl.cookie.BasicClientCookie;
21 22 import org.apache.http.impl.cookie.DefaultCookieSpec;
22 23 import org.apache.http.message.BasicHeader;
  24 +import org.apache.http.message.BasicNameValuePair;
23 25 import org.apache.http.protocol.HttpContext;
24 26 import org.apache.http.util.EntityUtils;
25 27 import org.springframework.lang.NonNull;
... ... @@ -608,74 +610,10 @@ public class RequestUtil {
608 610 }
609 611  
610 612 /**
611   - * 设置全局代理(更改系统属性方式)
612   - *
613   - * @throws IOException
614   - */
615   - private void setGlobalProxy() throws IOException {
616   - Properties prop = System.getProperties();
617   - // 设置http访问要使用的代理服务器的地址
618   - prop.setProperty("http.proxyHost", "183.45.78.31");
619   - // 设置http访问要使用的代理服务器的端口
620   - prop.setProperty("http.proxyPort", "8080");
621   - // 设置不需要通过代理服务器访问的主机,可以使用*通配符,多个地址用|分隔
622   - prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*");
623   - // 设置安全访问使用的代理服务器地址与端口
624   - // 它没有https.nonProxyHosts属性,它按照http.nonProxyHosts 中设置的规则访问
625   - prop.setProperty("https.proxyHost", "183.45.78.31");
626   - prop.setProperty("https.proxyPort", "443");
627   - // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机
628   - prop.setProperty("ftp.proxyHost", "183.45.78.31");
629   - prop.setProperty("ftp.proxyPort", "21");
630   - prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*");
631   - // socks代理服务器的地址与端口
632   - prop.setProperty("socksProxyHost", "183.45.78.31");
633   - prop.setProperty("socksProxyPort", "1080");
634   - // 设置登陆到代理服务器的用户名和密码
635   - Authenticator.setDefault(new MyAuthenticator("userName", "Password"));
636   - }
637   -
638   - private static void setProxy() throws IOException {
639   - URL url = new URL("http://www.baidu.com");
640   - // 创建代理服务器
641   - InetSocketAddress addr = new InetSocketAddress("202.116.64.93", 3969);
642   - // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理
643   - Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理
644   - //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码
645   - HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问
646   - InputStreamReader in = new InputStreamReader(connection.getInputStream());
647   - BufferedReader reader = new BufferedReader(in);
648   - while (true) {
649   - String s = reader.readLine();
650   - if (s != null) {
651   - System.out.println(s);
652   - }
653   - }
654   - }
655   -
656   - public static void main(String[] args) throws IOException {
657   - setProxy();
658   - }
659   -
660   - static class MyAuthenticator extends Authenticator {
661   - private String user = "";
662   - private String password = "";
663   -
664   - public MyAuthenticator(String user, String password) {
665   - this.user = user;
666   - this.password = password;
667   - }
668   -
669   - protected PasswordAuthentication getPasswordAuthentication() {
670   - return new PasswordAuthentication(user, password.toCharArray());
671   - }
672   - }
673   -
674   - /**
675 613 * 发送Post请求,参数拼接到url
676 614 *
677 615 * @param httpUrl 请求路径
678   - * @param map 请求参数
  616 + * @param map 请求参数
679 617 * @return
680 618 */
681 619 public static String sendPost(String httpUrl, Map<String, String> map) {
... ... @@ -846,4 +784,151 @@ public class RequestUtil {
846 784 return buffer.toString();
847 785 }
848 786  
  787 + /**
  788 + * 利用代理服务器请求数据
  789 + *
  790 + * @param apiURL 需要请求的接口地址
  791 + * @param params 请求接口传参
  792 + * @param proxyHost 代理服务器路径
  793 + * @param proxyPort 代理服务器端口号
  794 + * @return 返回结果
  795 + * @throws Exception
  796 + */
  797 + private static String proxyPOST(String apiURL, List<BasicNameValuePair> params, String proxyHost, Integer proxyPort) throws Exception {
  798 + String result;
  799 + HttpPost post = new HttpPost(apiURL);
  800 + CloseableHttpClient client = null;
  801 + CloseableHttpResponse response = null;
  802 + HttpEntity entity = null;
  803 + try {
  804 + post.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
  805 +
  806 + post.setHeader("X-Requested-With", "XMLHttpRequest");
  807 + post.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
  808 + // 表单请求参数
  809 + post.setEntity(new UrlEncodedFormEntity(params));
  810 + // 设置客户端超时时间
  811 + RequestConfig config = RequestConfig.custom()
  812 + .setSocketTimeout(50000)
  813 + .setConnectTimeout(50000)
  814 + .setConnectionRequestTimeout(50000)
  815 + .build();
  816 + HttpClientBuilder httpClientBuilder = HttpClients.custom()
  817 + .setDefaultRequestConfig(config);
  818 + log.info(proxyHost);
  819 + // 取代理服务器的ip和端口
  820 + HttpHost targetHost = new HttpHost(proxyHost, proxyPort);
  821 + CredentialsProvider credsProvider = new BasicCredentialsProvider();
  822 + /*credsProvider.setCredentials(
  823 + new AuthScope(targetHost.getHostName(), targetHost.getPort()),
  824 + new UsernamePasswordCredentials("", "")
  825 + );*/
  826 + httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
  827 + httpClientBuilder.setProxy(targetHost);
  828 + log.info("设置代理完成...");
  829 + client = httpClientBuilder.build();
  830 + log.info("获取客户端请求...");
  831 + // 执行POST请求
  832 + response = client.execute(post);
  833 + entity = response.getEntity();
  834 + int statusCode = response.getStatusLine().getStatusCode();
  835 + log.info("执行请求结束...响应状态码:" + statusCode);
  836 + result = EntityUtils.toString(entity, "UTF-8");
  837 + } catch (Exception e) {
  838 + post.abort();
  839 + throw e;
  840 + } finally {
  841 + EntityUtils.consume(entity);
  842 + // 关闭连接 释放资源
  843 + post.releaseConnection();
  844 + if (response != null) {
  845 + response.close();
  846 + }
  847 + if (client != null) {
  848 + client.close();
  849 + }
  850 + }
  851 + return result;
  852 + }
  853 +
  854 + /**
  855 + * 设置全局代理(更改系统属性方式)
  856 + *
  857 + * @throws IOException
  858 + */
  859 + private void setGlobalProxy() throws IOException {
  860 + Properties prop = System.getProperties();
  861 + // 设置http访问要使用的代理服务器的地址
  862 + prop.setProperty("http.proxyHost", "183.45.78.31");
  863 + // 设置http访问要使用的代理服务器的端口
  864 + prop.setProperty("http.proxyPort", "8080");
  865 + // 设置不需要通过代理服务器访问的主机,可以使用*通配符,多个地址用|分隔
  866 + prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*");
  867 + // 设置安全访问使用的代理服务器地址与端口
  868 + // 它没有https.nonProxyHosts属性,它按照http.nonProxyHosts 中设置的规则访问
  869 + prop.setProperty("https.proxyHost", "183.45.78.31");
  870 + prop.setProperty("https.proxyPort", "443");
  871 + // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机
  872 + prop.setProperty("ftp.proxyHost", "183.45.78.31");
  873 + prop.setProperty("ftp.proxyPort", "21");
  874 + prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*");
  875 + // socks代理服务器的地址与端口
  876 + prop.setProperty("socksProxyHost", "183.45.78.31");
  877 + prop.setProperty("socksProxyPort", "1080");
  878 + // 设置登陆到代理服务器的用户名和密码
  879 + Authenticator.setDefault(new MyAuthenticator("userName", "Password"));
  880 + }
  881 +
  882 + private static void setProxy() throws IOException {
  883 + URL url = new URL("http://www.baidu.com");
  884 + // 创建代理服务器
  885 + InetSocketAddress addr = new InetSocketAddress("27.159.66.41", 20263);
  886 + // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理
  887 + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理
  888 + //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码
  889 + HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问
  890 + InputStreamReader in = new InputStreamReader(connection.getInputStream());
  891 + BufferedReader reader = new BufferedReader(in);
  892 + while (true) {
  893 + String s = reader.readLine();
  894 + if (s != null) {
  895 + System.out.println(s);
  896 + }
  897 + }
  898 + }
  899 +
  900 + static class MyAuthenticator extends Authenticator {
  901 + private String user = "";
  902 + private String password = "";
  903 +
  904 + public MyAuthenticator(String user, String password) {
  905 + this.user = user;
  906 + this.password = password;
  907 + }
  908 +
  909 + protected PasswordAuthentication getPasswordAuthentication() {
  910 + return new PasswordAuthentication(user, password.toCharArray());
  911 + }
  912 + }
  913 +
  914 + public static void main(String[] args) throws Exception {
  915 + List<BasicNameValuePair> params = new ArrayList<>();
  916 + final BasicNameValuePair pair1 = new BasicNameValuePair("principalId", "NXXZX8888");
  917 + final BasicNameValuePair pair2 = new BasicNameValuePair("pcursor", "");
  918 + final BasicNameValuePair pair3 = new BasicNameValuePair("count", "150");
  919 + final BasicNameValuePair pair4 = new BasicNameValuePair("operationName", "playbackFeedsQuery");
  920 + params.add(pair1);
  921 + params.add(pair2);
  922 + params.add(pair3);
  923 + params.add(pair4);
  924 + final BasicNameValuePair pair5 = new BasicNameValuePair("variables", JSON.toJSONString(params));
  925 + List<BasicNameValuePair> params1 = new ArrayList<>();
  926 + final BasicNameValuePair pair6 = new BasicNameValuePair("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" +
  927 + " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " +
  928 + "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}");
  929 + params1.add(pair5);
  930 + params1.add(pair6);
  931 + System.out.println(proxyPOST("https://live.kuaishou.com/live_graphql", params1, "27.159.66.41", 20108));
  932 + }
  933 +
849 934 }
... ...
src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java
... ... @@ -29,11 +29,11 @@ public class ThreadPoolUtil {
29 29 synchronized (ThreadPoolUtil.class) {// threadPool == null, 加锁
30 30 if (threadPool == null) {
31 31 threadPool = new ThreadPoolExecutor(
  32 + 6,
32 33 12,
33   - 24,
34 34 60,
35 35 TimeUnit.SECONDS,
36   - new LinkedBlockingQueue<>(124),
  36 + new LinkedBlockingQueue<>(256),
37 37 new ThreadFactoryBuilder().setNamePrefix("capture-pool-").build(),
38 38 new ThreadPoolExecutor.DiscardPolicy());// 自行创建线程池, 并将创建好的线程池对象赋值给类的静态成员变量threadPool
39 39 }
... ...