third / fw-freya

Commit af4452ec5f742f5b8da4f41c0060159ec3222edd

Authored by 王明元
1 parent 24727067

2023年7月13日16:53:43 优化多线程账号加锁解锁逻辑

Inline Side-by-side
Showing 3 changed files with 211 additions and 43 deletions
db/freya.mv.db
View file @af4452e
No preview for this file type
db/freya.trace.db
  View file @af4452e
... ... @@ -42729,3 +42729,174 @@ Caused by: org.h2.jdbc.JdbcSQLNonTransientConnectionException: The connection wa
42729 42729 at org.h2.message.DbException.getJdbcSQLException(DbException.java:622)
42730 42730 at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
42731 42731 ... 18 more
  42732 +2023-07-05 14:55:50 jdbc[7749]: null
  42733 +org.h2.message.DbException: The connection was not closed by the application and is garbage collected [90018-200]
  42734 + at org.h2.message.DbException.get(DbException.java:205)
  42735 + at org.h2.message.DbException.get(DbException.java:181)
  42736 + at org.h2.message.DbException.get(DbException.java:170)
  42737 + at org.h2.jdbc.JdbcConnection.closeOld(JdbcConnection.java:245)
  42738 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:186)
  42739 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152)
  42740 + at org.h2.Driver.connect(Driver.java:69)
  42741 + at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138)
  42742 + at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:364)
  42743 + at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:206)
  42744 + at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:476)
  42745 + at com.zaxxer.hikari.pool.HikariPool.access$100(HikariPool.java:71)
  42746 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:726)
  42747 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:712)
  42748 + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
  42749 + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  42750 + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  42751 + at java.base/java.lang.Thread.run(Thread.java:829)
  42752 +Caused by: org.h2.jdbc.JdbcSQLNonTransientConnectionException: The connection was not closed by the application and is garbage collected [90018-200]
  42753 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:622)
  42754 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
  42755 + ... 18 more
  42756 +2023-07-05 14:55:50 jdbc[7749]: java.lang.Exception: Open Stack Trace
  42757 + at org.h2.util.CloseWatcher.register(CloseWatcher.java:100)
  42758 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:187)
  42759 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152)
  42760 + at org.h2.Driver.connect(Driver.java:69)
  42761 + at org.h2.util.JdbcUtils.getConnection(JdbcUtils.java:309)
  42762 + at org.h2.util.JdbcUtils.getConnection(JdbcUtils.java:289)
  42763 + at org.h2.server.web.WebServer.getConnection(WebServer.java:785)
  42764 + at org.h2.server.web.WebApp.login(WebApp.java:1008)
  42765 + at org.h2.server.web.WebApp.process(WebApp.java:223)
  42766 + at org.h2.server.web.WebApp.processRequest(WebApp.java:173)
  42767 + at org.h2.server.web.WebServlet.doGet(WebServlet.java:129)
  42768 + at org.h2.server.web.WebServlet.doPost(WebServlet.java:166)
  42769 + at javax.servlet.http.HttpServlet.service(HttpServlet.java:681)
  42770 + at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
  42771 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)
  42772 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42773 + at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
  42774 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42775 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42776 + at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
  42777 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42778 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42779 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42780 + at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
  42781 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42782 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42783 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42784 + at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
  42785 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42786 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42787 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42788 + at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)
  42789 + at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
  42790 + at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:540)
  42791 + at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)
  42792 + at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
  42793 + at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
  42794 + at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
  42795 + at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:382)
  42796 + at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
  42797 + at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:895)
  42798 + at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1722)
  42799 + at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
  42800 + at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
  42801 + at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
  42802 + at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
  42803 + at java.base/java.lang.Thread.run(Thread.java:829)
  42804 +
  42805 +org.h2.message.DbException: The connection was not closed by the application and is garbage collected [90018-200]
  42806 + at org.h2.message.DbException.get(DbException.java:205)
  42807 + at org.h2.message.DbException.get(DbException.java:181)
  42808 + at org.h2.message.DbException.get(DbException.java:170)
  42809 + at org.h2.jdbc.JdbcConnection.closeOld(JdbcConnection.java:245)
  42810 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:186)
  42811 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152)
  42812 + at org.h2.Driver.connect(Driver.java:69)
  42813 + at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138)
  42814 + at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:364)
  42815 + at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:206)
  42816 + at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:476)
  42817 + at com.zaxxer.hikari.pool.HikariPool.access$100(HikariPool.java:71)
  42818 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:726)
  42819 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:712)
  42820 + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
  42821 + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  42822 + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  42823 + at java.base/java.lang.Thread.run(Thread.java:829)
  42824 +Caused by: org.h2.jdbc.JdbcSQLNonTransientConnectionException: The connection was not closed by the application and is garbage collected [90018-200]
  42825 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:622)
  42826 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
  42827 + ... 18 more
  42828 +2023-07-05 17:33:19 jdbc[7806]: java.lang.Exception: Open Stack Trace
  42829 + at org.h2.util.CloseWatcher.register(CloseWatcher.java:100)
  42830 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:187)
  42831 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152)
  42832 + at org.h2.Driver.connect(Driver.java:69)
  42833 + at org.h2.util.JdbcUtils.getConnection(JdbcUtils.java:309)
  42834 + at org.h2.util.JdbcUtils.getConnection(JdbcUtils.java:289)
  42835 + at org.h2.server.web.WebServer.getConnection(WebServer.java:785)
  42836 + at org.h2.server.web.WebApp.login(WebApp.java:1008)
  42837 + at org.h2.server.web.WebApp.process(WebApp.java:223)
  42838 + at org.h2.server.web.WebApp.processRequest(WebApp.java:173)
  42839 + at org.h2.server.web.WebServlet.doGet(WebServlet.java:129)
  42840 + at org.h2.server.web.WebServlet.doPost(WebServlet.java:166)
  42841 + at javax.servlet.http.HttpServlet.service(HttpServlet.java:681)
  42842 + at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
  42843 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)
  42844 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42845 + at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
  42846 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42847 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42848 + at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
  42849 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42850 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42851 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42852 + at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
  42853 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42854 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42855 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42856 + at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
  42857 + at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
  42858 + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
  42859 + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
  42860 + at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)
  42861 + at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
  42862 + at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:540)
  42863 + at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)
  42864 + at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
  42865 + at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
  42866 + at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
  42867 + at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:382)
  42868 + at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
  42869 + at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:895)
  42870 + at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1722)
  42871 + at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
  42872 + at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
  42873 + at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
  42874 + at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
  42875 + at java.base/java.lang.Thread.run(Thread.java:829)
  42876 +
  42877 +org.h2.message.DbException: The connection was not closed by the application and is garbage collected [90018-200]
  42878 + at org.h2.message.DbException.get(DbException.java:205)
  42879 + at org.h2.message.DbException.get(DbException.java:181)
  42880 + at org.h2.message.DbException.get(DbException.java:170)
  42881 + at org.h2.jdbc.JdbcConnection.closeOld(JdbcConnection.java:245)
  42882 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:186)
  42883 + at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152)
  42884 + at org.h2.Driver.connect(Driver.java:69)
  42885 + at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138)
  42886 + at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:364)
  42887 + at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:206)
  42888 + at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:476)
  42889 + at com.zaxxer.hikari.pool.HikariPool.access$100(HikariPool.java:71)
  42890 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:726)
  42891 + at com.zaxxer.hikari.pool.HikariPool$PoolEntryCreator.call(HikariPool.java:712)
  42892 + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
  42893 + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  42894 + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  42895 + at java.base/java.lang.Thread.run(Thread.java:829)
  42896 +Caused by: org.h2.jdbc.JdbcSQLNonTransientConnectionException: The connection was not closed by the application and is garbage collected [90018-200]
  42897 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:622)
  42898 + at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
  42899 + ... 18 more
  42900 +2023-07-13 15:17:07 jdbc[13]: exception
  42901 +org.h2.jdbc.JdbcSQLSyntaxErrorException: Syntax error in SQL statement "DELETE *[*] FROM NS_SIG3"; expected "identifier"; SQL statement:
  42902 +delete * FROM NS_SIG3 [42001-200]
... ...
src/main/java/cn/fw/freya/service/CrawlBizService.java
  View file @af4452e
... ... @@ -107,51 +107,48 @@ public class CrawlBizService {
107 107 Integer type = account.getType();
108 108 String accountNo = account.getAccountNo();
109 109 String key = accountNo + "->" + type;
110   - AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
111   - CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
112   - autoExpireLocalCache.put(key, key, 3600 * 1000L);// 1小时自动过期
113   - // 抓取数据
114   - log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据");
115   - ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数
116   - if (Objects.isNull(accountMsg)) {
117   - this.doPushExpireAccount(account);
118   - autoExpireLocalCache.removeKey(key);
119   - return;
120   - }
121   - List<VideoPool> allVideoMsg = crawlStrategy.getAllVideoMsg(accountNo);// 获取所有视频信息
122   - if (Objects.isNull(allVideoMsg)) {
123   - this.doPushExpireAccount(account);
124   - autoExpireLocalCache.removeKey(key);
125   - return;
126   - }
127   - List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息
128   - if (Objects.isNull(yesterdayLiveMsg)) {
129   - this.doPushExpireAccount(account);
130   - autoExpireLocalCache.removeKey(key);
131   - return;
132   - }
133   - // 上报数据
134   - boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg);
135   - String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo);
136   - if (!reportAccountMsg) {
137   - log.error(format + "的账户信息失败");
138   - autoExpireLocalCache.removeKey(key);
139   - return;
140   - }
141   - boolean reportVideo = this.doReportVideo(account, allVideoMsg);
142   - if (!reportVideo) {
143   - log.error(format + "的视频数据失败");
144   - autoExpireLocalCache.removeKey(key);
145   - return;
146   - }
147   - boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
148   - if (!reportLive) {
149   - log.error(format + "的直播数据失败");
  110 + try {
  111 + AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
  112 + CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
  113 + autoExpireLocalCache.put(key, key, 3600 * 1000L);// 1小时自动过期
  114 + // 抓取数据
  115 + log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据");
  116 + ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数
  117 + if (Objects.isNull(accountMsg)) {
  118 + this.doPushExpireAccount(account);
  119 + return;
  120 + }
  121 + List<VideoPool> allVideoMsg = crawlStrategy.getAllVideoMsg(accountNo);// 获取所有视频信息
  122 + if (Objects.isNull(allVideoMsg)) {
  123 + this.doPushExpireAccount(account);
  124 + return;
  125 + }
  126 + List<LivePool> yesterdayLiveMsg = crawlStrategy.getYesterdayLiveMsg(accountNo);// 获取昨日直播信息
  127 + if (Objects.isNull(yesterdayLiveMsg)) {
  128 + this.doPushExpireAccount(account);
  129 + return;
  130 + }
  131 + // 上报数据
  132 + boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg);
  133 + String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo);
  134 + if (!reportAccountMsg) {
  135 + log.error(format + "的账户信息失败");
  136 + return;
  137 + }
  138 + boolean reportVideo = this.doReportVideo(account, allVideoMsg);
  139 + if (!reportVideo) {
  140 + log.error(format + "的视频数据失败");
  141 + return;
  142 + }
  143 + boolean reportLive = this.doReportLive(account, yesterdayLiveMsg);
  144 + if (!reportLive) {
  145 + log.error(format + "的直播数据失败");
  146 + return;
  147 + }
  148 + this.afterCrawl(accountService.getById(account.getId()));
  149 + } finally {
150 150 autoExpireLocalCache.removeKey(key);
151   - return;
152 151 }
153   - autoExpireLocalCache.removeKey(key);
154   - this.afterCrawl(accountService.getById(account.getId()));
155 152 }
156 153  
157 154 /**
... ...