third / fw-freya

package cn.fw.freya.service.crawl.impl;

import cn.fw.freya.common.BusinessException;
import cn.fw.freya.dao.AccountDao;
import cn.fw.freya.dao.LivePoolDao;
import cn.fw.freya.dao.VideoPoolDao;
import cn.fw.freya.enums.AccountTypeEnum;
import cn.fw.freya.enums.DataTypeEnum;
import cn.fw.freya.enums.ResourceTypeEnum;
import cn.fw.freya.model.data.Account;
import cn.fw.freya.model.data.FwCookie;
import cn.fw.freya.model.data.ResponseReceived;
import cn.fw.freya.model.data.pool.LivePool;
import cn.fw.freya.model.data.pool.VideoPool;
import cn.fw.freya.model.dto.rpc.ReportAccountDto;
import cn.fw.freya.service.crawl.CrawlStrategy;
import cn.fw.freya.service.data.AccountService;
import cn.fw.freya.service.rpc.AccountRpcService;
import cn.fw.freya.utils.DateUtil;
import cn.fw.freya.utils.JsonUtils;
import cn.fw.freya.utils.PublicUtil;
import cn.fw.freya.utils.RequestUtil;
import cn.fw.freya.utils.http.HttpConfig;
import cn.fw.freya.utils.http.HttpCookies;
import cn.fw.freya.utils.http.HttpHeader;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.alibaba.fastjson.parser.Feature;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.logging.log4j.util.PropertiesUtil;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.springframework.context.SmartLifecycle;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.support.PropertiesLoaderUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import javax.annotation.Resource;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
import java.util.stream.Collectors;

/**
 * @author null
 * @date 2021-11-11 17:15
 * @description 快手数据抓取
 */
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("Duplicates")
public class KuaiShouCrawl implements CrawlStrategy, SmartLifecycle {

    private boolean isRunning = false;
    private final VideoPoolDao videoPoolDao;
    private final LivePoolDao livePoolDao;
    private final AccountDao accountDao;
    private final Common common;
    public final static ConcurrentHashMap<String, WebDriver> DRIVER_MAP = new ConcurrentHashMap<>();
    private final AccountService accountService;
    private final String playbackBaseUrl = "https://live.kuaishou.com/playback/";
    private final ConcurrentHashMap<String, String> sig3Map = new ConcurrentHashMap<>();
    private final AccountRpcService accountRpcService;

    @Resource(name = "wmyThreadPool")
    private ThreadPoolExecutor threadPoolExecutor;

    @Override
    public AccountTypeEnum getType() {
        return AccountTypeEnum.KS;
    }

    /**
     * 设置sig3Map
     */
    public boolean setSig3Map() {
        this.sig3Map.clear();
        final List<Account> accountList = accountDao.getAllKSAccount();
        accountList.forEach(item ->
                Arrays.stream(DataTypeEnum.values()).forEach(item1 ->
                        threadPoolExecutor.execute(() -> this.task(item.getAccountNo(), item1.getValue()))
                )
        );
        return true;
    }

    /**
     * 获取类型签名task
     *
     * @param accountNo 账户号
     * @param dataType  数据类型
     */
    public void task(String accountNo, Integer dataType) {
        String key = accountNo + "#" + dataType;
        final String ns_sig3 = this.getNS_sig3(accountNo, dataType, true);
        if (Objects.nonNull(ns_sig3))
            sig3Map.put(key, ns_sig3);
        else
            accountRpcService.pushExpireAccount(accountNo, this.getType().getValue());
    }

    /**
     * 停止设置sig3Map线程池
     */
    public boolean stopSetSig3Map() {
        threadPoolExecutor.shutdownNow();
        return true;
    }

    /**
     * 清空sig3Map
     */
    public boolean cleanSig3Map() {
        sig3Map.clear();
        return true;
    }

    /**
     * 获取sig3Map
     */
    public String getSig3Map() {
        return JSON.toJSONString(sig3Map);
    }

    /**
     * 设置sig3Map
     */
    public boolean setMapFromString(String jsonStr, boolean interior) {
        String jsonString;
        if (interior)
            jsonString = jsonStr;
        else
            jsonString = JSON.parseObject(jsonStr).getString("jsonStr");
        HashMap<String, String> map = JSON.parseObject(jsonString, new TypeReference<>() {
        }, Feature.OrderedField);
        map.forEach(sig3Map::put);
        return true;
    }

    /**
     * 将现有的签名信息map写入到配置文件中
     */
    public boolean writeMapToProperties() {
        final HashMap<String, String> props = new HashMap<>();
        props.put("KSAccountNS_sig", JSON.toJSONString(sig3Map));
        updateProperties("NS_sig3Msg.properties", props);
        return true;
    }

    /**
     * 获取快手登录二维码
     *
     * @param accountNo 账户号
     * @return
     */
    @Override
    public String preLogin(String accountNo) {
        WebDriver driver = DRIVER_MAP.get(accountNo);
        try {
            if (Objects.isNull(driver)) {
                driver = common.createDriver();
                DRIVER_MAP.put(accountNo, driver);
            }
            driver.get("https://cp.kuaishou.com/profile");// 打开指定的页面
            new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//a[text()='立即登录']"))).click();// 获取'登录'按钮元素, 单击
            new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[text()='扫码登录']"))).click();
            WebElement qrCodeEle = new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[starts-with(@class,'qrcode')]/img[1]")));// 获取网页'登录二维码'元素对象
            return qrCodeEle.getAttribute("src");// 返回对象src属性对应的值
        } catch (Exception e) {
            log.error("获取快手登录二维码发生错误", e);
            if (Objects.nonNull(driver)) {
                driver.quit();
                DRIVER_MAP.remove(accountNo);
            }
            throw new BusinessException(e.getMessage());
        }
    }

    /**
     * 随意请求一个接口, 获取到浏览器cookies, 保存
     *
     * @param accountNo 账户号
     * @return
     * @throws BusinessException
     */
    @Override
    public boolean doLogin(String accountNo) throws BusinessException {
        WebDriver driver = DRIVER_MAP.get(accountNo);
        if (Objects.isNull(driver)) {
            throw new BusinessException("登陆校验失败，请重新尝试");
        }
        final WebElement element;
        try {
            element = new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[contains(text(),'快手号：') or contains(text(),'用户 ID：')]")));
        } catch (Exception e) {
            this.exitBrowser(accountNo, null);
            throw new BusinessException("网络异常，或该人员暂未扫码登录");
        }
        if (Objects.nonNull(element)) {
            int subLength = 0;
            final String accountText = element.getText();
            if (accountText.startsWith("用户")) {
                subLength = 6;
            } else if (accountText.startsWith("快手号")) {
                subLength = 4;
            }
            if (accountNo.equals(accountText.substring(subLength))) {
                Integer type = this.getType().getValue();
                common.saveCookie(driver, accountNo, type);// 保存该用户的cookies
                accountService.updateAccountCookiesStatus(accountNo, type, true);
                common.whenLoginCheckAccountExist(accountNo, type);
                this.exitBrowser(accountNo, null);
                return true;
            } else {
                this.exitBrowser(accountNo, null);
                throw new BusinessException("实际扫码人员与指定扫码人员不同");
            }
        }
        throw new BusinessException("登陆校验失败，请重新尝试");
    }

    /**
     * 获取所有视频作品信息
     *
     * @param accountNo 账户号
     * @throws IOException
     */
    @Override
    @Transactional
    public List<VideoPool> getAllVideoMsg(String accountNo) throws IOException {
        final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundVideo)) {
            return hasFoundVideo;
        }
        HttpCookies cookies = HttpCookies.custom();
        CookieStore cookieStore = new BasicCookieStore();
        cookies.setCookieStore(cookieStore);
        Date previousDay = DateUtil.getPreviousDay(new Date());
        final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.VIDEO.getValue(), false);
        if (Objects.isNull(ns_sig3))
            return null;
        Map<String, Object> params = new LinkedHashMap<>();
        params.put("count", 10);
        params.put("page", 1);
        //params.put("total", 9007199000000000L + new Random().nextInt(999999999));
        params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo));
        HttpConfig config = HttpConfig.custom()
                .url("https://cp.kuaishou.com/rest/cp/creator/pc/analysis/photo/list?__NS_sig3=" + ns_sig3)
                .context(cookies.getContext())
                .json(JsonUtils.objectToJson(params))
                .headers(HttpHeader
                        .defaultHeader()
                        .contentType("application/json")
                        .host("cp.kuaishou.com")
                        .cookie(this.getUserCookies(accountNo))
                        .build()
                );
        String res = RequestUtil.post(config);// 发送POST请求
        log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, res));
        final JSONObject response = JSONObject.parseObject(res);
        if (this.verifyCookies(response)) {
            return null;
        }
        if (!StringUtils.hasText(res)) {
            throw new BusinessException("调用快手[视频]接口失败");
        }
        if (Objects.equals(response.getInteger("result"), 500002)) {
            threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.VIDEO.getValue()));
            throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
        }
        JSONArray videoJsonArray = Optional.ofNullable(Optional.ofNullable(response.getJSONObject("data")).orElse(new JSONObject()).getJSONArray("photoList")).orElse(new JSONArray());
        videoPoolDao.deleteByAccountNoAndDate(accountNo, previousDay, AccountTypeEnum.KS.getValue(), ResourceTypeEnum.VIDEO.getValue());
        // 视频数据存库
        List<VideoPool> videoPoolList = new ArrayList<>(videoJsonArray.size());
        videoJsonArray.forEach(item -> {
            JSONObject obj = (JSONObject) item;
            assert obj != null;
            //if (!obj.getBoolean("privateStatus")) {
            /**
             * -commentCount: 0
             * -completePlayRate: 0.087
             * -cover: "https://p3.a.yximgs.com/upic/2021/12/03/19/BMjAyMTEyMDMxOTI1MzZfMjU2MTc2NDMyMV82MjAxNDgwNzM0NV8xXzM=_Bfec1ec6582987d6b69e1f3e792481bee.jpg?tag=1-1641559034-nil-0-crkuvfievz-fd836eb1f65cbf16&clientCacheKey=3x4gpy8ts3encgm.jpg&di=6eb8a1b7&bp=10000"
             * -duration: 12766
             * -increaseFansCount: 0
             * -likeCount: 6
             * -playCount: 293
             * playUrl: "https://txmov2.a.kwimgs.com/upic/2021/12/03/19/BMjAyMTEyMDMxOTI1MzZfMjU2MTc2NDMyMV82MjAxNDgwNzM0NV8xXzM=_b_B4d2eed08f38cfd34420a92ccbddabbc1.mp4?tag=1-1641559034-w-0-zjc9fjmnnl-7bda305bd4f9cbf3&clientCacheKey=3x4gpy8ts3encgm_b.mp4&tt=b&di=6eb8a1b7&bp=10000"
             * privateStatus: false
             * -publishTime: "2021-12-03 19:25:58"
             * -shareCount: 0
             * -title: "长安汽车，年终盛典#CS75plus 恭喜李哥和陆姐喜提爱车😁感谢大哥对小马的信任"
             * -workId: "3x4gpy8ts3encgm"
             */
            int fullPlayCount;
            try {
                final BigDecimal[] bigDecimals = obj.getBigDecimal("completePlayRate")
                        .multiply(obj.getBigDecimal("playCount"))
                        .divideAndRemainder(BigDecimal.ONE);
                fullPlayCount = bigDecimals[0].intValue();
                if (bigDecimals[1].compareTo(BigDecimal.ZERO) > 0) {
                    fullPlayCount++;
                }
            } catch (Exception e) {
                log.error("[快手]计算[fullPlayCount]指标发生错误", e);
                fullPlayCount = 0;
            }
            videoPoolList.add(VideoPool.builder()
                    .videoId(Optional.ofNullable(obj.getString("workId")).orElse(""))//
                    .title(Optional.ofNullable(obj.getString("title")).orElse(""))//
                    .preview(obj.getString("cover"))//
                    .playCount(Optional.ofNullable(obj.getInteger("playCount")).orElse(0))//
                    .likeCount(Optional.ofNullable(obj.getInteger("likeCount")).orElse(0))//
                    .commentCount(Optional.ofNullable(obj.getInteger("commentCount")).orElse(0))//
                    .accountNo(accountNo)
                    .reportDate(new Date())
                    .publishTime(PublicUtil.parseDate(obj.getString("publishTime")))//
                    .videoUrl("https://www.kuaishou.com/short-video/" + obj.getString("workId"))
                    .type(this.getType().getValue())
                    .resourceType(ResourceTypeEnum.VIDEO.getValue())
                    .fullPlayCount(fullPlayCount)
                    .duration(Optional.ofNullable(obj.getLong("duration")).orElse(0L) / 1000d)
                    .newFansUserCnt(Optional.ofNullable(obj.getInteger("increaseFansCount")).orElse(0))
                    .shareCount(Optional.ofNullable(obj.getInteger("shareCount")).orElse(0))
                    .build()
            );
            //}
        });
        try {
            if (videoPoolList.size() > 0) {
                videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
            } else {
                final VideoPool nullVideo = VideoPool.builder()
                        .accountNo(accountNo)
                        .type(this.getType().getValue())
                        .resourceType(ResourceTypeEnum.VIDEO.getValue())
                        .reportDate(new Date())
                        .build();
                videoPoolDao.save(nullVideo);
                final ArrayList<VideoPool> nullList = new ArrayList<>();
                //nullList.add(nullVideo);
                return nullList;
            }
        } catch (Exception e) {
            log.error(LocalDate.now() + " 执行保存" + accountNo + "的快手视频数据失败", e);
        }
        return videoPoolList;
    }

    /**
     * 获取昨天所有直播信息
     *
     * @param accountNo 账户号
     * @throws Exception
     */
    @Override
    @Transactional
    public List<LivePool> getYesterdayLiveMsg(String accountNo) throws IOException {
        final List<LivePool> hasFoundLive = common.getHasFoundLive(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundLive)) {
            return hasFoundLive;
        }
        HttpCookies cookies = HttpCookies.custom();
        CookieStore cookieStore = new BasicCookieStore();
        cookies.setCookieStore(cookieStore);
        Date previousDay = DateUtil.getPreviousDay(new Date());
        Date endTime = DateUtil.getThisDayMaxTime(previousDay);
        Date startTime = DateUtil.getThisDayMinTime(previousDay);
        //Date startTime = DateUtil.getThisDayMinTime(new Date(previousDay.getTime() - 7 * 24 * 3600 * 1000L));// 补数据使用
        final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.LIVE.getValue(), false);
        if (Objects.isNull(ns_sig3))
            return null;
        Map<String, Object> params = new LinkedHashMap<>();
        params.put("memberId", this.getUserId(accountNo));
        params.put("endTime", endTime.getTime());
        params.put("startTime", startTime.getTime() - 6 * 24 * 3600 * 1000L);
        params.put("dateType", 2);
        params.put("count", 50);
        params.put("page", 1);
        params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo));
        HttpConfig config = HttpConfig.custom()
                .url("https://cp.kuaishou.com/rest/cp/creator/analysis/live/pc/detail?__NS_sig3=" + ns_sig3)
                .context(cookies.getContext())
                .json(JsonUtils.objectToJson(params))
                .headers(HttpHeader
                        .defaultHeader()
                        .contentType("application/json")
                        .host("cp.kuaishou.com")
                        .cookie(this.getUserCookies(accountNo))
                        .build()
                );
        String res = RequestUtil.post(config);// 发送POST请求
        log.info(String.format("%s [%s]平台账户号为: %s的直播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, res));
        final JSONObject response = JSONObject.parseObject(res);
        if (this.verifyCookies(response)) {
            return null;
        }
        if (!StringUtils.hasText(res)) {
            throw new BusinessException("调用快手[视频]接口失败");
        }
        if (Objects.equals(response.getInteger("result"), 500002)) {
            threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.LIVE.getValue()));
            throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
        }
        JSONObject dataJSONObject = response.getJSONObject("data");
        JSONArray dataJSONArray = dataJSONObject.getJSONArray("details");
        /*List<JSONObject> collect = new ArrayList<>();
        if (!CollectionUtils.isEmpty(dataJSONArray)) {
            JSONArray userLivePlayback;
            Account account = accountDao.findByPhoneNoAndType(accountNo, this.getType().getValue());// 获取账号实体
            String playbackSearchKey = account.getPlaybackSearchKey();
            if (Objects.nonNull(playbackSearchKey))
                userLivePlayback = this.getUserLivePlayback(playbackSearchKey);
            else
                userLivePlayback = this.getUserLivePlayback(accountNo);
            collect = userLivePlayback
                    .stream()
                    .filter(item -> {
                        JSONObject obj = (JSONObject) item;
                        final Date createTime = obj.getDate("createTime");
                        return createTime.compareTo(startTime) >= 0 && createTime.compareTo(endTime) <= 0;
                    })
                    .map(item -> {
                        JSONObject obj = (JSONObject) item;
                        final Integer durationSeconds = obj.getInteger("duration");
                        obj.put("duration", BigDecimal.valueOf(durationSeconds).divide(BigDecimal.valueOf(60), 1, RoundingMode.HALF_UP).doubleValue());
                        obj.put("startTime", obj.getLong("createTime") - durationSeconds * 1000);
                        return obj;
                    })
                    .collect(Collectors.toList());
        }*/
        List<LivePool> livePoolList = new ArrayList<>();
        assert dataJSONArray != null;
        dataJSONArray.forEach(item -> {
            final JSONObject obj = (JSONObject) item;
            /**
             * authorId: 2561764321
             * -commentUv: 1
             * govId: 0
             * -likeCnt: 241
             * likeUv: 3
             * liveCnt: 0
             * -liveCover: "https://tx2.a.kwimgs.com/uhead/AB/2021/12/04/17/BMjAyMTEyMDQxNzU3MjRfMjU2MTc2NDMyMV84ODUwMzU5NzMwX2x2.jpg"
             * -liveDuration: 80.1
             * -liveStreamId: 8850359730
             * -liveTime: 1638611842193
             * -liveTitle: "长安汽车 年终盛典"
             * -maxConcurrentUv: 57
             * -newFansCnt: 0
             * pDate: null
             * -playUv: 165
             * -receiveAmount: 0
             * -sendGiftUv: 0
             * -shareUv: 1
             * userHead: "https://tx2.a.kwimgs.com/uhead/AB/2021/11/11/12/BMjAyMTExMTExMjUwMzZfMjU2MTc2NDMyMV8xX2hkMjg5XzMyOA==_s.jpg"
             * -userName: "长安汽车。小明聊聊车"
             */
            livePoolList.add(LivePool.builder()
                    .type(this.getType().getValue())
                    .accountNo(accountNo)
                    .reportDate(new Date())
                    .commentUserCnt(Optional.ofNullable(obj.getInteger("commentUv")).orElse(0))
                    .consumeUserCnt(Optional.ofNullable(obj.getInteger("sendGiftUv")).orElse(0))
                    .duration(Optional.of(obj.getDouble("liveDuration")).orElse(0d))
                    .income(Optional.ofNullable(obj.getDouble("receiveAmount")).orElse(0d))
                    .newFansUserCnt(Optional.ofNullable(obj.getInteger("newFansCnt")).orElse(0))
                    .openTime(new Date(Optional.ofNullable(obj.getLong("liveTime")).orElse(0L)))
                    .roomCoverImage(Optional.ofNullable(obj.getString("liveCover")).orElse(""))
                    .playbackUrl(null)
                    .roomId(Optional.ofNullable(obj.getString("liveStreamId")).orElse(""))
                    .roomName(Optional.ofNullable(obj.getString("liveTitle")).orElse(""))
                    //.score(obj.getInteger("score"))
                    //.watchCnt(obj.getInteger("watch_cnt"))
                    .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("maxConcurrentUv")).orElse(0))
                    .watchUserCnt(Optional.ofNullable(obj.getInteger("playUv")).orElse(0))
                    .getPlaybackFailTimes(0)

                    .shareCnt(Optional.ofNullable(obj.getInteger("shareUv")).orElse(0))
                    .likeCnt(Optional.ofNullable(obj.getInteger("likeCnt")).orElse(0))
                    .userNick(Optional.ofNullable(obj.getString("userName")).orElse(""))
                    .build());
        });
        try {
            if (livePoolList.size() > 0) {
                livePoolDao.saveAll(livePoolList.stream()
                        .filter(item -> item.getOpenTime().compareTo(startTime) >= 0 && item.getOpenTime().compareTo(endTime) <= 0)
                        .collect(Collectors.toList())
                );
            } else {
                log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的快手直播数据");
                final LivePool nullLive = LivePool.builder()
                        .type(this.getType().getValue())
                        .accountNo(accountNo)
                        .reportDate(new Date())
                        .build();
                livePoolDao.save(nullLive);
                //livePoolList.add(nullLive);
            }
        } catch (Exception e) {
            log.error(LocalDate.now() + " 执行保存" + accountNo + "的快手直播数据失败", e);
        }
        return livePoolList;
    }

    public JSONObject setPlaybackUrl(LivePool dbLive, List<JSONObject> collect, Double durationThreshold) {
        JSONObject object = new JSONObject();
        object.put("coverUrl", null);
        object.put("playbackUrl", null);
        if (CollectionUtils.isEmpty(collect))
            return object;
        final String accountNo = dbLive.getAccountNo();
        final Double liveDuration = dbLive.getDuration();// 数据接口返回直播时长
        final Long liveStartTimeStamp = dbLive.getOpenTime().getTime();// 数据接口返回开播时间戳
        if (liveDuration >= durationThreshold) {// 直播时长大于等于durationThreshold指定的分钟才记录回放信息
            JSONObject playbackMsg;
            if (Objects.equals(collect.size(), 1)) {
                playbackMsg = collect.get(0);
                Double duration = playbackMsg.getDouble("duration");// 回放信息返回直播时长
                final Long startTimeStamp = playbackMsg.getLong("startTime");// 回放信息返回直播开始时间戳
                final double timeSubAbs = Math.abs(liveDuration - duration);
                final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
                // (Objects.equals(liveDuration, duration) || (timeSubAbs < 5))->说明时长几乎相等
                // (liveStartSub < 2)->说明开播时间几乎一样
                if ((!((Objects.equals(liveDuration, duration) ||
                        (timeSubAbs < 5)) && liveStartSub < 10)) &&
                        !(startTimeStamp >= liveStartTimeStamp && startTimeStamp <= liveStartTimeStamp + liveDuration * 60 * 1000 && duration >= durationThreshold)) {
                    playbackMsg = null;
                    log.info(String.format("%s [%s]平台账户号为: %s的直播回放数据不匹配!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
                }
            } else {
                List<JSONObject> collect1 = collect.stream().filter(item1 -> {
                    Double duration = item1.getDouble("duration");
                    final Long startTimeStamp = item1.getLong("startTime");// 回放信息返回直播开始时间戳
                    final double timeSubAbs = Math.abs(liveDuration - duration);
                    final double liveStartSub = BigDecimal.valueOf(Math.abs(liveStartTimeStamp - startTimeStamp)).divide(BigDecimal.valueOf(60 * 1000), 1, RoundingMode.HALF_UP).doubleValue();
                    return (Objects.equals(liveDuration, duration) || timeSubAbs < 5) && liveStartSub < 10;
                }).collect(Collectors.toList());
                if (Objects.equals(collect1.size(), 1)) {
                    playbackMsg = collect1.get(0);
                } else {
                    playbackMsg = null;
                    log.info(String.format("%s [%s]平台账户号为: %s的直播找到多条回放数据!!!", LocalDateTime.now(), this.getType().getName(), accountNo));
                }
            }
            if (Objects.nonNull(playbackMsg)) {
                object.put("coverUrl", playbackMsg.getString("coverUrl"));
                object.put("playbackUrl", this.playbackBaseUrl + playbackMsg.getString("productId"));
            } else {
                object.put("hasInvalidPlayback", true);
            }
        }
        return object;
    }

    public JSONArray getUserLivePlayback(String searchKey) {
        JSONArray objects = new JSONArray();
        if (Objects.isNull(searchKey))
            return objects;
        Map<String, Object> params = new LinkedHashMap<>();
        Map<String, Object> params1 = new LinkedHashMap<>();
        params1.put("principalId", searchKey);
        params1.put("pcursor", "");
        params1.put("count", 150);
        params.put("operationName", "playbackFeedsQuery");
        params.put("variables", params1);
        params.put("query", "query playbackFeedsQuery($principalId: String, $pcursor: String, $count: Int)" +
                " {playbackFeeds(principalId: $principalId, pcursor: $pcursor, count: $count) {pcursor list " +
                "{productId, coverUrl, caption, createTime, duration, viewCount, likeCount, commentCount, likeStatus, __typename, baseUrl, manifestUrl}}}"
        );
        /*HttpConfig config = HttpConfig.custom()
                .encoding(java.nio.charset.StandardCharsets.UTF_8.displayName())
                .url("https://live.kuaishou.com/live_graphql")
                .json(JSON.toJSONString(params))
                .headers(HttpHeader
                        .defaultHeader()
                        .other("Origin", "https://live.kuaishou.com")// 非必须
                        .referer("https://live.kuaishou.com/profile/" + searchKey)// 非必须
                        .userAgent(this.getUserAgent(searchKey))// 非必须
                        .cookie(this.getRandomUserCookies())
                        .build()
                );
        String res = RequestUtil.post(config);*/
        final String res = RequestUtil.sendPostBody("https://live.kuaishou.com/live_graphql", params);
        log.info(String.format("%s [%s]平台账户号为: %s的回播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, res));
        if (!StringUtils.hasText(res)) {
            return objects;
        }
        JSONObject resObj = JSONObject.parseObject(res);
        try {
            return resObj.getJSONObject("data").getJSONObject("playbackFeeds").getJSONArray("list");
        } catch (Exception e) {
            log.error(String.format("%s 抓取[%s]平台账户号为: %s的直播回放数据出现异常!!! 异常信息为: %s", LocalDateTime.now(), this.getType().getName(), searchKey, e.getMessage()));
            return objects;
        }
    }

    /**
     * 更新账户粉丝数
     *
     * @param accountNo
     */
    @Override
    @Transactional
    public ReportAccountDto updateAccountMsg(String accountNo) throws IOException {
        final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundAccountMsg)) {
            return hasFoundAccountMsg;
        }
        HttpCookies cookies = HttpCookies.custom();
        CookieStore cookieStore = new BasicCookieStore();
        cookies.setCookieStore(cookieStore);
        Map<String, Object> params = new HashMap<>();
        params.put("kuaishou.web.cp.api_ph", this.getWebApiPh(accountNo));
        final String ns_sig3 = this.getNS_sig3(accountNo, DataTypeEnum.FANS.getValue(), false);
        if (Objects.isNull(ns_sig3))
            return null;
        HttpConfig config = HttpConfig.custom()
                .url("https://cp.kuaishou.com/rest/cp/creator/pc/home/infoV2?__NS_sig3=" + ns_sig3)
                .context(cookies.getContext())
                .json(JsonUtils.objectToJson(params))
                .headers(HttpHeader
                        .defaultHeader()
                        .other("Origin", "https://cp.kuaishou.com")// 非必须
                        .referer("https://cp.kuaishou.com/profile")// 非必须
                        .userAgent(this.getUserAgent(accountNo))// 非必须
                        .cookie(this.getUserCookies(accountNo))
                        .build()
                );
        String res = RequestUtil.post(config);// 发送POST请求
        final JSONObject response = JSONObject.parseObject(res);
        if (this.verifyCookies(response)) {
            return null;
        }
        if (!StringUtils.hasText(res)) {
            throw new BusinessException("调用快手[直播]接口失败");
        }
        if (Objects.equals(response.getInteger("result"), 500002)) {
            threadPoolExecutor.execute(() -> this.task(accountNo, DataTypeEnum.FANS.getValue()));
            throw new BusinessException("获取数据失败, 尝试重新获取sig3签名信息");
        }
        JSONObject data = response.getJSONObject("data");
        ReportAccountDto reportAccountDto = ReportAccountDto.builder()
                .fansCnt(data.getInteger("fansCnt"))
                .followCnt(data.getInteger("followCnt"))
                .likeCnt(data.getInteger("likeCnt"))
                .accountId(data.getString("userId"))
                .userOtherId(data.getString("userKwaiId"))
                .accountName(data.getString("userName"))
                .build();
        final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
        if (Objects.nonNull(account))
            accountDao.updateMsg(account.getId(), reportAccountDto.getAccountName(), reportAccountDto.getFansCnt(), new Date());
        else
            accountDao.save(Account.builder()
                    .cookiesStatus(true)
                    .accountNo(accountNo)
                    .type(this.getType().getValue())
                    .fansCnt(reportAccountDto.getFansCnt())
                    .accountName(reportAccountDto.getAccountName())
                    .reportDate(new Date())
                    .done(false)
                    .build());
        return reportAccountDto;
    }

    /**
     * 进入指定账户号的创作者平台首页
     *
     * @param accountNo 快手账户号
     */
    public void loginIndex(String accountNo) {
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getKSDriver(accountNo, uuid);
        String targetUrl = "https://cp.kuaishou.com/profile";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        try {
            WebElement startExperienceButton = new WebDriverWait(driver, 6, 2000).until(driver1 ->
                    driver1.findElement(By.xpath("//*[@id='driver-popover-item']/div[4]/span/button[2]")));
            if (Objects.nonNull(startExperienceButton)) {
                for (int i = 0; i < 3; i++) {
                    new WebDriverWait(driver, 6, 500).until(driver1 ->
                            driver1.findElement(By.xpath("//*[@id='driver-popover-item']/div[4]/span/button[2]"))).click();
                    this.waitFor(0.75);
                }
            }
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException(e.getMessage());
        }
        this.exitBrowser(accountNo, uuid);
    }

    /**
     * 进入指定账户号的创作者平台首页
     *
     * @param accountNo 快手账户号
     * @param type      密钥类型(1:粉丝, 2:短视频, 3:直播)
     * @param retryGet  是否重新获取
     */
    public String getNS_sig3(String accountNo, Integer type, boolean retryGet) {
        final String key = accountNo + "#" + type;
        String NS_sig3;
        if (!retryGet) {
            NS_sig3 = sig3Map.get(key);
            if (StringUtils.hasText(NS_sig3))
                return NS_sig3;
        }
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getKSDriver(accountNo, uuid);
        String targetUrl = null;
        String dataUrl = null;// 数据接口地址
        if (Objects.equals(type, DataTypeEnum.FANS.getValue())) {
            targetUrl = "https://cp.kuaishou.com/profile";
            dataUrl = "https://cp.kuaishou.com/rest/cp/creator/pc/home/infoV2";
        } else if (Objects.equals(type, DataTypeEnum.VIDEO.getValue())) {
            targetUrl = "https://cp.kuaishou.com/statistics/works";
            dataUrl = "https://cp.kuaishou.com/rest/cp/creator/pc/analysis/photo/list";
        } else if (Objects.equals(type, DataTypeEnum.LIVE.getValue())) {
            targetUrl = "https://cp.kuaishou.com/statistics/live";
            dataUrl = "https://cp.kuaishou.com/rest/cp/creator/analysis/live/pc/detail";
        }
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
        if (Objects.equals(type, DataTypeEnum.FANS.getValue())) {
            driver.get("https://cp.kuaishou.com/article/manage/video");
            LockSupport.parkNanos(TimeUnit.SECONDS.toNanos(5));
            if (Objects.equals(targetUrl, driver.getCurrentUrl())) {
                this.exitBrowser(accountNo, uuid);
                return null;
            }
        }
        if (Objects.equals("https://cp.kuaishou.com/profile", driver.getCurrentUrl())) {// 页面未跳转到视频/直播数据页面
            this.exitBrowser(accountNo, uuid);
            return null;
        }
        try {
            final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
            for (ResponseReceived item : responseReceivedEvents) {
                String str = this.getDataUrl(item, dataUrl);
                if (StringUtils.hasText(str)) {
                    this.exitBrowser(accountNo, uuid);
                    String[] split = str.split("=");
                    NS_sig3 = split[1];
                    sig3Map.put(key, NS_sig3);
                    return NS_sig3;
                }
            }
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException(e.getMessage());
        }
        this.exitBrowser(accountNo, uuid);
        return null;
    }

    /**
     * 读取http日志获取数据接口全路径地址
     *
     * @param responseReceived 收到的响应
     * @param dataUrl          数据接口地址
     * @return
     */
    public String getDataUrl(ResponseReceived responseReceived, String dataUrl) {
        String baseUrl = JSONObject.parseObject(responseReceived.getResponse()).getString("url");
        boolean notStaticFiles = !baseUrl.endsWith(".png")
                && !baseUrl.endsWith(".jpg")
                && !baseUrl.endsWith(".css")
                && !baseUrl.endsWith(".ico")
                && !baseUrl.endsWith(".js")
                && !baseUrl.endsWith(".gif");
        if (notStaticFiles && baseUrl.contains(dataUrl))
            return baseUrl;
        return null;
    }

    /**
     * 将快手账户的cookies放入driver中
     *
     * @param accountNo
     * @return
     */
    private WebDriver getKSDriver(String accountNo, String uuid) {
        WebDriver driver = common.createDriver();
        try {
            driver.get("https://cp.kuaishou.com/profile");
            Set<Cookie> cookieSet = common.loadCookie(accountNo, this.getType().getValue())
                    .stream()
                    .map(FwCookie::seleniumCookie)
                    .collect(Collectors.toSet());
            cookieSet.forEach(item -> driver.manage().addCookie(item));// 给driver加载账户的cookies;
            DRIVER_MAP.put(accountNo + "#" + uuid, driver);
            return driver;
        } catch (Exception e) {
            driver.close();
            throw new BusinessException("创建快手驱动发生异常: " + e.getMessage());
        }
    }

    /**
     * 根据账户号查到用户cookies, 并组装成String形式返回
     *
     * @param accountNo 账户号
     * @return
     */
    public String getUserCookies(String accountNo) {
        final List<FwCookie> cookies = common.loadCookie(accountNo, this.getType().getValue());
        StringBuffer sb = new StringBuffer();
        sb.append("clientid=3;");
        cookies.forEach(item -> sb.append(item.getName()).append("=").append(item.getValue()).append(";"));
        return sb.toString();
    }

    /**
     * 根据账户号获取 kuaishou.web.cp.api_ph
     *
     * @param accountNo 账户号
     * @return
     */
    public String getWebApiPh(String accountNo) {
        final List<FwCookie> cookies = common.loadCookie(accountNo, this.getType().getValue());
        for (FwCookie cookie : cookies) {
            if (Objects.equals(cookie.getName(), "kuaishou.web.cp.api_ph"))
                return cookie.getValue();
        }
        return null;
    }

    /**
     * 根据账户号获取 账户id
     *
     * @param accountNo 账户号
     * @return
     */
    public Long getUserId(String accountNo) {
        final List<FwCookie> cookies = common.loadCookie(accountNo, this.getType().getValue());
        for (FwCookie cookie : cookies) {
            if (Objects.equals(cookie.getName(), "userId"))
                return Long.valueOf(cookie.getValue());
        }
        return null;
    }

    /**
     * 获取随机用户cookies, 并组装成String形式返回
     *
     * @return
     */
    public String getRandomUserCookies() {
        final List<FwCookie> cookies = common.loadCookie(common.getRandomUserByType(this.getType().getValue()).getAccountNo(), this.getType().getValue());
        StringBuffer sb = new StringBuffer();
        sb.append("clientid=3;");
        cookies.forEach(item -> {
            sb.append(item.getName()).append("=").append(item.getValue()).append(";");
        });
        return sb.toString();
    }

    /**
     * 验证账户cookies是否失效
     *
     * @param response
     * @return
     */
    public boolean verifyCookies(JSONObject response) {
        if (Objects.nonNull(response)) {
            try {
                final Integer resultCode = response.getInteger("result");
                final String loginUrl = response.getString("loginUrl");
                if (Objects.equals(109, resultCode) || Objects.nonNull(loginUrl)) {
                    return true;
                } else {
                    return false;
                }
            } catch (Exception e) {
                throw new BusinessException("验证快手cookies发生异常: " + e.getMessage());
            }
        } else {
            log.error("调用快手平台接口未接收到返回值");
            throw new BusinessException("调用快手平台接口未接收到返回值");
        }
    }

    /**
     * 退出浏览器
     *
     * @param accountNo 账户号
     * @return
     */
    public boolean exitBrowser(String accountNo, String uuid) {
        String key = accountNo + (Objects.nonNull(uuid) ? "#" + uuid : "");
        WebDriver driver = DRIVER_MAP.get(key);
        if (Objects.isNull(driver)) {
            return true;
        }
        driver.close();
        DRIVER_MAP.remove(key);
        return true;
    }

    @Override
    public int getPhase() {
        return 1;// 默认为0
    }

    @Override
    public boolean isAutoStartup() {
        return true;// 默认为false
    }

    @Override
    public void stop(Runnable callback) {
        log.info("springIOC停止, 将签名信息从Map中导出到配置文件");
        callback.run();
        isRunning = false;
        this.writeMapToProperties();
    }

    @Override
    public void start() {
        log.info("springIOC启动, 将签名信息从配置文件读出并导入到Map中");
        isRunning = true;
        Properties props;
        try {
            props = PropertiesLoaderUtils.loadAllProperties("NS_sig3Msg.properties");
            for (Object key : props.keySet()) {
                if (Objects.equals(key, "KSAccountNS_sig"))
                    this.setMapFromString(props.get(key).toString(), true);
            }
        } catch (IOException e) {
            log.error(e.getMessage());
        }
    }

    @Override
    public void stop() {
        log.info("stop");
        isRunning = false;
    }

    @Override
    public boolean isRunning() {
        return isRunning;
    }

    public void updateProperties(String fileName, Map<String, String> keyValueMap) {
        String filePath = Objects.requireNonNull(PropertiesUtil.class.getClassLoader().getResource(fileName)).getFile();
        Properties props;
        BufferedWriter bw = null;
        try {
            filePath = URLDecoder.decode(filePath, StandardCharsets.UTF_8);
            log.debug("updateProperties propertiesPath:" + filePath);
            props = PropertiesLoaderUtils.loadProperties(new ClassPathResource(fileName));
            log.debug("updateProperties old:" + props);
            bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath)));// 写入属性文件
            //props.clear();// 清空旧的文件
            for (String key : keyValueMap.keySet()) {
                props.setProperty(key, keyValueMap.get(key));
            }
            log.debug("updateProperties new:" + props);
            props.store(bw, "");
        } catch (IOException e) {
            log.error(e.getMessage());
        } finally {
            try {
                assert bw != null;
                bw.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}