Commit e17d27076650fb3ddc456790be399b174f4108b4
1 parent
6c0810cc
2022年6月29日13:03:00 增加从jsonStr设置sig3Map接口
Showing
3 changed files
with
25 additions
and
5 deletions
src/main/java/cn/fw/freya/controller/KSController.java
@@ -6,9 +6,7 @@ import cn.fw.freya.service.crawl.impl.KuaiShouCrawl; | @@ -6,9 +6,7 @@ import cn.fw.freya.service.crawl.impl.KuaiShouCrawl; | ||
6 | import com.alibaba.fastjson.JSONObject; | 6 | import com.alibaba.fastjson.JSONObject; |
7 | import lombok.RequiredArgsConstructor; | 7 | import lombok.RequiredArgsConstructor; |
8 | import org.springframework.validation.annotation.Validated; | 8 | import org.springframework.validation.annotation.Validated; |
9 | -import org.springframework.web.bind.annotation.GetMapping; | ||
10 | -import org.springframework.web.bind.annotation.RequestMapping; | ||
11 | -import org.springframework.web.bind.annotation.RestController; | 9 | +import org.springframework.web.bind.annotation.*; |
12 | 10 | ||
13 | import javax.servlet.http.HttpServletRequest; | 11 | import javax.servlet.http.HttpServletRequest; |
14 | import java.io.IOException; | 12 | import java.io.IOException; |
@@ -95,6 +93,15 @@ public class KSController { | @@ -95,6 +93,15 @@ public class KSController { | ||
95 | } | 93 | } |
96 | 94 | ||
97 | /** | 95 | /** |
96 | + * 设置setSig3Map | ||
97 | + * | ||
98 | + */ | ||
99 | + @PostMapping("/setMapFromString") | ||
100 | + public boolean setMapFromString(@RequestBody String jsonStr) { | ||
101 | + return kuaishouCrawl.setMapFromString(jsonStr); | ||
102 | + } | ||
103 | + | ||
104 | + /** | ||
98 | * 停止设置setSig3Map | 105 | * 停止设置setSig3Map |
99 | * | 106 | * |
100 | */ | 107 | */ |
src/main/java/cn/fw/freya/service/crawl/impl/KuaiShouCrawl.java
@@ -24,6 +24,8 @@ import cn.fw.freya.utils.http.HttpHeader; | @@ -24,6 +24,8 @@ import cn.fw.freya.utils.http.HttpHeader; | ||
24 | import com.alibaba.fastjson.JSON; | 24 | import com.alibaba.fastjson.JSON; |
25 | import com.alibaba.fastjson.JSONArray; | 25 | import com.alibaba.fastjson.JSONArray; |
26 | import com.alibaba.fastjson.JSONObject; | 26 | import com.alibaba.fastjson.JSONObject; |
27 | +import com.alibaba.fastjson.TypeReference; | ||
28 | +import com.alibaba.fastjson.parser.Feature; | ||
27 | import lombok.RequiredArgsConstructor; | 29 | import lombok.RequiredArgsConstructor; |
28 | import lombok.extern.slf4j.Slf4j; | 30 | import lombok.extern.slf4j.Slf4j; |
29 | import org.apache.http.client.CookieStore; | 31 | import org.apache.http.client.CookieStore; |
@@ -113,6 +115,16 @@ public class KuaiShouCrawl implements CrawlStrategy { | @@ -113,6 +115,16 @@ public class KuaiShouCrawl implements CrawlStrategy { | ||
113 | } | 115 | } |
114 | 116 | ||
115 | /** | 117 | /** |
118 | + * 设置sig3Map | ||
119 | + */ | ||
120 | + public boolean setMapFromString(String jsonStr) { | ||
121 | + HashMap<String, String> map = JSON.parseObject(JSON.parseObject(jsonStr).getString("jsonStr"), new TypeReference<>() { | ||
122 | + }, Feature.OrderedField); | ||
123 | + map.forEach(sig3Map::put); | ||
124 | + return true; | ||
125 | + } | ||
126 | + | ||
127 | + /** | ||
116 | * 获取快手登录二维码 | 128 | * 获取快手登录二维码 |
117 | * | 129 | * |
118 | * @param accountNo 账户号 | 130 | * @param accountNo 账户号 |
src/main/java/cn/fw/freya/task/DataCaptureTask.java
@@ -16,6 +16,7 @@ import com.alibaba.fastjson.JSONObject; | @@ -16,6 +16,7 @@ import com.alibaba.fastjson.JSONObject; | ||
16 | import lombok.RequiredArgsConstructor; | 16 | import lombok.RequiredArgsConstructor; |
17 | import lombok.extern.slf4j.Slf4j; | 17 | import lombok.extern.slf4j.Slf4j; |
18 | import org.springframework.context.ApplicationEventPublisher; | 18 | import org.springframework.context.ApplicationEventPublisher; |
19 | +import org.springframework.scheduling.annotation.Scheduled; | ||
19 | import org.springframework.stereotype.Component; | 20 | import org.springframework.stereotype.Component; |
20 | import org.springframework.util.CollectionUtils; | 21 | import org.springframework.util.CollectionUtils; |
21 | 22 | ||
@@ -50,7 +51,7 @@ public class DataCaptureTask { | @@ -50,7 +51,7 @@ public class DataCaptureTask { | ||
50 | /** | 51 | /** |
51 | * 每分钟执行多线程同时抓取数据 | 52 | * 每分钟执行多线程同时抓取数据 |
52 | */ | 53 | */ |
53 | - //@Scheduled(cron = "0 0/1 * * * ?") | 54 | + @Scheduled(cron = "0 0/1 * * * ?") |
54 | public void capture() { | 55 | public void capture() { |
55 | final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); | 56 | final ThreadPoolExecutor threadPoolExecutor = ThreadPoolUtil.getThreadPool(); |
56 | BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 | 57 | BlockingQueue<Runnable> queue = threadPoolExecutor.getQueue();// 获取工作队列 |
@@ -96,7 +97,7 @@ public class DataCaptureTask { | @@ -96,7 +97,7 @@ public class DataCaptureTask { | ||
96 | /** | 97 | /** |
97 | * 每2分钟执行抓取数据 | 98 | * 每2分钟执行抓取数据 |
98 | */ | 99 | */ |
99 | - //@Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) | 100 | + @Scheduled(fixedRate = 2 * 60 * 1000, initialDelay = 5000) |
100 | public void captureLivePlayback() { | 101 | public void captureLivePlayback() { |
101 | Double durationThreshold = 60d;// 设置直播时长阈值 | 102 | Double durationThreshold = 60d;// 设置直播时长阈值 |
102 | final Random random = new Random(); | 103 | final Random random = new Random(); |