Commit b8357e80d0ccd6068b3a847fae2a7d66b0687ccc
1 parent
28d3e509
2022年5月30日18:20:09 调整线程池参数, 减小内存炸鸡概率
Showing
2 changed files
with
157 additions
and
72 deletions
src/main/java/cn/fw/freya/utils/RequestUtil.java
... | ... | @@ -2,13 +2,14 @@ package cn.fw.freya.utils; |
2 | 2 | |
3 | 3 | import cn.fw.freya.common.Callback; |
4 | 4 | import cn.fw.freya.utils.http.*; |
5 | +import com.alibaba.fastjson.JSON; | |
5 | 6 | import com.alibaba.fastjson.JSONObject; |
6 | 7 | import lombok.extern.slf4j.Slf4j; |
7 | -import org.apache.http.Header; | |
8 | -import org.apache.http.HttpEntity; | |
9 | -import org.apache.http.HttpResponse; | |
10 | -import org.apache.http.NameValuePair; | |
8 | +import org.apache.http.*; | |
9 | +import org.apache.http.client.CredentialsProvider; | |
11 | 10 | import org.apache.http.client.HttpClient; |
11 | +import org.apache.http.client.config.RequestConfig; | |
12 | +import org.apache.http.client.entity.UrlEncodedFormEntity; | |
12 | 13 | import org.apache.http.client.methods.*; |
13 | 14 | import org.apache.http.client.utils.DateUtils; |
14 | 15 | import org.apache.http.config.Lookup; |
... | ... | @@ -16,10 +17,11 @@ import org.apache.http.cookie.Cookie; |
16 | 17 | import org.apache.http.cookie.CookieOrigin; |
17 | 18 | import org.apache.http.cookie.CookieSpecProvider; |
18 | 19 | import org.apache.http.cookie.MalformedCookieException; |
19 | -import org.apache.http.impl.client.CookieSpecRegistries; | |
20 | +import org.apache.http.impl.client.*; | |
20 | 21 | import org.apache.http.impl.cookie.BasicClientCookie; |
21 | 22 | import org.apache.http.impl.cookie.DefaultCookieSpec; |
22 | 23 | import org.apache.http.message.BasicHeader; |
24 | +import org.apache.http.message.BasicNameValuePair; | |
23 | 25 | import org.apache.http.protocol.HttpContext; |
24 | 26 | import org.apache.http.util.EntityUtils; |
25 | 27 | import org.springframework.lang.NonNull; |
... | ... | @@ -608,74 +610,10 @@ public class RequestUtil { |
608 | 610 | } |
609 | 611 | |
610 | 612 | /** |
611 | - * 设置全局代理(更改系统属性方式) | |
612 | - * | |
613 | - * @throws IOException | |
614 | - */ | |
615 | - private void setGlobalProxy() throws IOException { | |
616 | - Properties prop = System.getProperties(); | |
617 | - // 设置http访问要使用的代理服务器的地址 | |
618 | - prop.setProperty("http.proxyHost", "183.45.78.31"); | |
619 | - // 设置http访问要使用的代理服务器的端口 | |
620 | - prop.setProperty("http.proxyPort", "8080"); | |
621 | - // 设置不需要通过代理服务器访问的主机,可以使用*通配符,多个地址用|分隔 | |
622 | - prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); | |
623 | - // 设置安全访问使用的代理服务器地址与端口 | |
624 | - // 它没有https.nonProxyHosts属性,它按照http.nonProxyHosts 中设置的规则访问 | |
625 | - prop.setProperty("https.proxyHost", "183.45.78.31"); | |
626 | - prop.setProperty("https.proxyPort", "443"); | |
627 | - // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机 | |
628 | - prop.setProperty("ftp.proxyHost", "183.45.78.31"); | |
629 | - prop.setProperty("ftp.proxyPort", "21"); | |
630 | - prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); | |
631 | - // socks代理服务器的地址与端口 | |
632 | - prop.setProperty("socksProxyHost", "183.45.78.31"); | |
633 | - prop.setProperty("socksProxyPort", "1080"); | |
634 | - // 设置登陆到代理服务器的用户名和密码 | |
635 | - Authenticator.setDefault(new MyAuthenticator("userName", "Password")); | |
636 | - } | |
637 | - | |
638 | - private static void setProxy() throws IOException { | |
639 | - URL url = new URL("http://www.baidu.com"); | |
640 | - // 创建代理服务器 | |
641 | - InetSocketAddress addr = new InetSocketAddress("202.116.64.93", 3969); | |
642 | - // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理 | |
643 | - Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理 | |
644 | - //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码 | |
645 | - HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问 | |
646 | - InputStreamReader in = new InputStreamReader(connection.getInputStream()); | |
647 | - BufferedReader reader = new BufferedReader(in); | |
648 | - while (true) { | |
649 | - String s = reader.readLine(); | |
650 | - if (s != null) { | |
651 | - System.out.println(s); | |
652 | - } | |
653 | - } | |
654 | - } | |
655 | - | |
656 | - public static void main(String[] args) throws IOException { | |
657 | - setProxy(); | |
658 | - } | |
659 | - | |
660 | - static class MyAuthenticator extends Authenticator { | |
661 | - private String user = ""; | |
662 | - private String password = ""; | |
663 | - | |
664 | - public MyAuthenticator(String user, String password) { | |
665 | - this.user = user; | |
666 | - this.password = password; | |
667 | - } | |
668 | - | |
669 | - protected PasswordAuthentication getPasswordAuthentication() { | |
670 | - return new PasswordAuthentication(user, password.toCharArray()); | |
671 | - } | |
672 | - } | |
673 | - | |
674 | - /** | |
675 | 613 | * 发送Post请求,参数拼接到url |
676 | 614 | * |
677 | 615 | * @param httpUrl 请求路径 |
678 | - * @param map 请求参数 | |
616 | + * @param map 请求参数 | |
679 | 617 | * @return |
680 | 618 | */ |
681 | 619 | public static String sendPost(String httpUrl, Map<String, String> map) { |
... | ... | @@ -846,4 +784,151 @@ public class RequestUtil { |
846 | 784 | return buffer.toString(); |
847 | 785 | } |
848 | 786 | |
787 | + /** | |
788 | + * 利用代理服务器请求数据 | |
789 | + * | |
790 | + * @param apiURL 需要请求的接口地址 | |
791 | + * @param params 请求接口传参 | |
792 | + * @param proxyHost 代理服务器路径 | |
793 | + * @param proxyPort 代理服务器端口号 | |
794 | + * @return 返回结果 | |
795 | + * @throws Exception | |
796 | + */ | |
797 | + private static String proxyPOST(String apiURL, List<BasicNameValuePair> params, String proxyHost, Integer proxyPort) throws Exception { | |
798 | + String result; | |
799 | + HttpPost post = new HttpPost(apiURL); | |
800 | + CloseableHttpClient client = null; | |
801 | + CloseableHttpResponse response = null; | |
802 | + HttpEntity entity = null; | |
803 | + try { | |
804 | + post.setHeader("Accept", "application/json, text/javascript, */*; q=0.01"); | |
805 | + | |
806 | + post.setHeader("X-Requested-With", "XMLHttpRequest"); | |
807 | + post.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); | |
808 | + // 表单请求参数 | |
809 | + post.setEntity(new UrlEncodedFormEntity(params)); | |
810 | + // 设置客户端超时时间 | |
811 | + RequestConfig config = RequestConfig.custom() | |
812 | + .setSocketTimeout(50000) | |
813 | + .setConnectTimeout(50000) | |
814 | + .setConnectionRequestTimeout(50000) | |
815 | + .build(); | |
816 | + HttpClientBuilder httpClientBuilder = HttpClients.custom() | |
817 | + .setDefaultRequestConfig(config); | |
818 | + log.info(proxyHost); | |
819 | + // 取代理服务器的ip和端口 | |
820 | + HttpHost targetHost = new HttpHost(proxyHost, proxyPort); | |
821 | + CredentialsProvider credsProvider = new BasicCredentialsProvider(); | |
822 | + /*credsProvider.setCredentials( | |
823 | + new AuthScope(targetHost.getHostName(), targetHost.getPort()), | |
824 | + new UsernamePasswordCredentials("", "") | |
825 | + );*/ | |
826 | + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); | |
827 | + httpClientBuilder.setProxy(targetHost); | |
828 | + log.info("设置代理完成..."); | |
829 | + client = httpClientBuilder.build(); | |
830 | + log.info("获取客户端请求..."); | |
831 | + // 执行POST请求 | |
832 | + response = client.execute(post); | |
833 | + entity = response.getEntity(); | |
834 | + int statusCode = response.getStatusLine().getStatusCode(); | |
835 | + log.info("执行请求结束...响应状态码:" + statusCode); | |
836 | + result = EntityUtils.toString(entity, "UTF-8"); | |
837 | + } catch (Exception e) { | |
838 | + post.abort(); | |
839 | + throw e; | |
840 | + } finally { | |
841 | + EntityUtils.consume(entity); | |
842 | + // 关闭连接 释放资源 | |
843 | + post.releaseConnection(); | |
844 | + if (response != null) { | |
845 | + response.close(); | |
846 | + } | |
847 | + if (client != null) { | |
848 | + client.close(); | |
849 | + } | |
850 | + } | |
851 | + return result; | |
852 | + } | |
853 | + | |
854 | + /** | |
855 | + * 设置全局代理(更改系统属性方式) | |
856 | + * | |
857 | + * @throws IOException | |
858 | + */ | |
859 | + private void setGlobalProxy() throws IOException { | |
860 | + Properties prop = System.getProperties(); | |
861 | + // 设置http访问要使用的代理服务器的地址 | |
862 | + prop.setProperty("http.proxyHost", "183.45.78.31"); | |
863 | + // 设置http访问要使用的代理服务器的端口 | |
864 | + prop.setProperty("http.proxyPort", "8080"); | |
865 | + // 设置不需要通过代理服务器访问的主机,可以使用*通配符,多个地址用|分隔 | |
866 | + prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); | |
867 | + // 设置安全访问使用的代理服务器地址与端口 | |
868 | + // 它没有https.nonProxyHosts属性,它按照http.nonProxyHosts 中设置的规则访问 | |
869 | + prop.setProperty("https.proxyHost", "183.45.78.31"); | |
870 | + prop.setProperty("https.proxyPort", "443"); | |
871 | + // 使用ftp代理服务器的主机、端口以及不需要使用ftp代理服务器的主机 | |
872 | + prop.setProperty("ftp.proxyHost", "183.45.78.31"); | |
873 | + prop.setProperty("ftp.proxyPort", "21"); | |
874 | + prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); | |
875 | + // socks代理服务器的地址与端口 | |
876 | + prop.setProperty("socksProxyHost", "183.45.78.31"); | |
877 | + prop.setProperty("socksProxyPort", "1080"); | |
878 | + // 设置登陆到代理服务器的用户名和密码 | |
879 | + Authenticator.setDefault(new MyAuthenticator("userName", "Password")); | |
880 | + } | |
881 | + | |
882 | + private static void setProxy() throws IOException { | |
883 | + URL url = new URL("http://www.baidu.com"); | |
884 | + // 创建代理服务器 | |
885 | + InetSocketAddress addr = new InetSocketAddress("27.159.66.41", 20263); | |
886 | + // Proxy proxy = new Proxy(Proxy.Type.SOCKS, addr); // Socket 代理 | |
887 | + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http 代理 | |
888 | + //Authenticator.setDefault(new MyAuthenticator("username", "password"));// 设置代理的用户和密码 | |
889 | + HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);// 设置代理访问 | |
890 | + InputStreamReader in = new InputStreamReader(connection.getInputStream()); | |
891 | + BufferedReader reader = new BufferedReader(in); | |
892 | + while (true) { | |
893 | + String s = reader.readLine(); | |
894 | + if (s != null) { | |
895 | + System.out.println(s); | |
896 | + } | |
897 | + } | |
898 | + } | |
899 | + | |
900 | + static class MyAuthenticator extends Authenticator { | |
901 | + private String user = ""; | |
902 | + private String password = ""; | |
903 | + | |
904 | + public MyAuthenticator(String user, String password) { | |
905 | + this.user = user; | |
906 | + this.password = password; | |
907 | + } | |
908 | + | |
909 | + protected PasswordAuthentication getPasswordAuthentication() { | |
910 | + return new PasswordAuthentication(user, password.toCharArray()); | |
911 | + } | |
912 | + } | |
913 | + | |
914 | + public static void main(String[] args) throws Exception { | |
915 | + List<BasicNameValuePair> params = new ArrayList<>(); | |
916 | + final BasicNameValuePair pair1 = new BasicNameValuePair("principalId", "NXXZX8888"); | |
917 | + final BasicNameValuePair pair2 = new BasicNameValuePair("pcursor", ""); | |
918 | + final BasicNameValuePair pair3 = new BasicNameValuePair("count", "150"); | |
919 | + final BasicNameValuePair pair4 = new BasicNameValuePair("operationName", "playbackFeedsQuery"); | |
920 | + params.add(pair1); | |
921 | + params.add(pair2); | |
922 | + params.add(pair3); | |
923 | + params.add(pair4); | |
924 | + final BasicNameValuePair pair5 = new BasicNameValuePair("variables", JSON.toJSONString(params)); | |
925 | + List<BasicNameValuePair> params1 = new ArrayList<>(); | |
926 | + final BasicNameValuePair pair6 = new BasicNameValuePair("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" + | |
927 | + " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " + | |
928 | + "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"); | |
929 | + params1.add(pair5); | |
930 | + params1.add(pair6); | |
931 | + System.out.println(proxyPOST("https://live.kuaishou.com/live_graphql", params1, "27.159.66.41", 20108)); | |
932 | + } | |
933 | + | |
849 | 934 | } | ... | ... |
src/main/java/cn/fw/freya/utils/ThreadPoolUtil.java
... | ... | @@ -29,11 +29,11 @@ public class ThreadPoolUtil { |
29 | 29 | synchronized (ThreadPoolUtil.class) {// threadPool == null, 加锁 |
30 | 30 | if (threadPool == null) { |
31 | 31 | threadPool = new ThreadPoolExecutor( |
32 | + 6, | |
32 | 33 | 12, |
33 | - 24, | |
34 | 34 | 60, |
35 | 35 | TimeUnit.SECONDS, |
36 | - new LinkedBlockingQueue<>(124), | |
36 | + new LinkedBlockingQueue<>(256), | |
37 | 37 | new ThreadFactoryBuilder().setNamePrefix("capture-pool-").build(), |
38 | 38 | new ThreadPoolExecutor.DiscardPolicy());// 自行创建线程池, 并将创建好的线程池对象赋值给类的静态成员变量threadPool |
39 | 39 | } | ... | ... |