third / fw-freya

Commit 573ef45b211e3003f53ad86d597864cbbc770c3c

Authored by 王明元
1 parent af4452ec

2023年7月13日17:07:13 优化日志打印

Inline Side-by-side
Showing 3 changed files with 5 additions and 48 deletions
db/freya.mv.db
View file @573ef45
No preview for this file type
src/main/java/cn/fw/freya/service/CrawlBizService.java
  View file @573ef45
... ... @@ -101,7 +101,7 @@ public class CrawlBizService {
101 101 /**
102 102 * 执行抓取数据
103 103 *
104   - * @param account
  104 + * @param account 待抓取账号实体
105 105 */
106 106 public void crawlData(Account account) throws IOException {
107 107 Integer type = account.getType();
... ... @@ -111,8 +111,8 @@ public class CrawlBizService {
111 111 AccountTypeEnum typeEnum = AccountTypeEnum.getEnumByValue(type);
112 112 CrawlStrategy crawlStrategy = crawlStrategyMap.get(typeEnum);
113 113 autoExpireLocalCache.put(key, key, 3600 * 1000L);// 1小时自动过期
114   - // 抓取数据
115   - log.info("线程: " + Thread.currentThread().getName() + " 开始抓取数据");
  114 + /* 抓取数据 */
  115 + log.info("{} 线程开始抓取{}: {} 的数据", Thread.currentThread().getName(), typeEnum.getName(), accountNo);
116 116 ReportAccountDto accountMsg = crawlStrategy.updateAccountMsg(accountNo);// 更新粉丝数
117 117 if (Objects.isNull(accountMsg)) {
118 118 this.doPushExpireAccount(account);
... ... @@ -128,7 +128,7 @@ public class CrawlBizService {
128 128 this.doPushExpireAccount(account);
129 129 return;
130 130 }
131   - // 上报数据
  131 + /* 上报数据 */
132 132 boolean reportAccountMsg = this.doReportAccountMsg(account, accountMsg);
133 133 String format = String.format("上报[%s]平台, 账户号为: %s", AccountTypeEnum.getNameByValue(account.getType()), accountNo);
134 134 if (!reportAccountMsg) {
... ...
src/main/java/cn/fw/freya/task/DataCaptureTask.java
  View file @573ef45
... ... @@ -55,7 +55,7 @@ public class DataCaptureTask {
55 55 ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool();
56 56 BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列
57 57 int activeCount = threadPoolExecutor.getActiveCount();// 获取正在执行的线程数
58   - log.info("当前队列中待执行任务数：" + queue.size() + "\t 正在执行任务的线程数：" + activeCount);
  58 + log.info("当前队列中待执行任务数：{}\t正在执行任务的线程数：{}", queue.size(), activeCount);
59 59 List<Account> waitCrawlAccount = accountService.findUnCrawlAccount();// 获取今日还未完成抓取数据的账号
60 60 if (CollectionUtils.isEmpty(waitCrawlAccount)) return;
61 61 for (Account account : waitCrawlAccount) {
... ... @@ -71,49 +71,6 @@ public class DataCaptureTask {
71 71 });
72 72 }
73 73 }
74   - /*public void capture() {
75   - ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool();
76   - BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列
77   - int activeCount = threadPoolExecutor.getActiveCount();// 获取正在执行的线程数
78   - log.info("当前队列中待执行任务数：" + queue.size() + "\t 正在执行任务的线程数：" + activeCount);
79   - List<Account> waitCrawlAccount = accountService.findUnCrawlAccount();// 获取今日还未完成抓取数据的账号
80   - if (CollectionUtils.isEmpty(waitCrawlAccount)) return;
81   - // 线程池运行多个线程同时抓取多个账号数据
82   - for (int i = 0; i < waitCrawlAccount.size(); i++) {
83   - Account currentAccount = waitCrawlAccount.get(i);
84   - String key = currentAccount.getAccountNo() + "->" + currentAccount.getType();
85   - if (StringUtils.hasText(autoExpireLocalCache.get(key)))
86   - continue;
87   - Account nextAccount;
88   - if (i < waitCrawlAccount.size() - 1) {
89   - nextAccount = waitCrawlAccount.get(i + 1);
90   - } else {
91   - nextAccount = new Account();
92   - }
93   - threadPoolExecutor.execute(() -> {
94   - try {
95   - crawlBizService.crawlData(currentAccount);
96   - } catch (IOException e) {
97   - log.error(String.format("多线程抓取数据发生IOException, 异常信息为: %s", e.getMessage()));
98   - }
99   - });
100   - Integer accountType = currentAccount.getType();// 当前账户类型
101   - Integer nextAccountType = Optional.ofNullable(nextAccount.getType()).orElse(-1);// 下一个账户类型
102   - String nextAccountNo = Optional.ofNullable(nextAccount.getAccountNo()).orElse("");// 下一个账户号
103   - Date todayDayMinTime = DateUtil.dayBegin(new Date());
104   - if (accountType.equals(2) && nextAccountType.equals(2)) {
105   - if (!Objects.equals(nextAccountNo, "") && !Objects.equals(nextAccountType, -1) &&
106   - (Objects.nonNull(common.getHasFoundAccountMsg(nextAccountNo, 2, todayDayMinTime)) &&
107   - Objects.nonNull(common.getHasFoundVideo(nextAccountNo, 2, todayDayMinTime)) &&
108   - Objects.nonNull(common.getHasFoundLive(nextAccountNo, 2, todayDayMinTime))
109   - )) {
110   - // 下一个账号为有效抖音号, 且之前已经拿到过该账号全部的数据
111   - } else {
112   - LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(20));
113   - }
114   - }
115   - }
116   - }*/
117 74  
118 75 /**
119 76 * 每2分钟执行抓取数据
... ...