third / fw-freya

package cn.fw.freya.service.crawl.impl;

import cn.fw.freya.common.BusinessException;
import cn.fw.freya.dao.AccountDao;
import cn.fw.freya.dao.LiveOverviewDao;
import cn.fw.freya.dao.LivePoolDao;
import cn.fw.freya.dao.VideoPoolDao;
import cn.fw.freya.enums.AccountTypeEnum;
import cn.fw.freya.enums.ResourceTypeEnum;
import cn.fw.freya.model.data.Account;
import cn.fw.freya.model.data.FwCookie;
import cn.fw.freya.model.data.LiveOverview;
import cn.fw.freya.model.data.ResponseReceived;
import cn.fw.freya.model.data.pool.LivePool;
import cn.fw.freya.model.data.pool.VideoPool;
import cn.fw.freya.model.dto.rpc.ReportAccountDto;
import cn.fw.freya.service.crawl.CrawlStrategy;
import cn.fw.freya.service.data.AccountService;
import cn.fw.freya.utils.DateUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.openqa.selenium.*;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;

import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

/**
 * @author null
 * @date 2021-11-12 11:32
 * @description
 */
@Slf4j
@Service
@RequiredArgsConstructor
@SuppressWarnings("Duplicates")
public class DouYinCrawl implements CrawlStrategy {

    private final VideoPoolDao videoPoolDao;
    private final LiveOverviewDao liveDataDao;
    private final LivePoolDao livePoolDao;
    private final AccountDao accountDao;
    public final static ConcurrentHashMap<String, WebDriver> DRIVER_MAP = new ConcurrentHashMap<>();
    private final Common common;
    private final AccountService accountService;
    private final int WAIT_SECONDS = 5;

    @Override
    public AccountTypeEnum getType() {
        return AccountTypeEnum.DY;
    }

    @Override
    public String preLogin(String accountNo) {
        WebDriver driver = DRIVER_MAP.get(accountNo);
        try {
            if (Objects.isNull(driver)) {
                driver = common.createDriver();
                DRIVER_MAP.put(accountNo, driver);
            }
            driver.get("https://creator.douyin.com/");// 打开指定的页面
            new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[@class='creator-header-nav']//span[@class='login']"))).click();// 获取'登录'按钮元素, 单击
            new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//span[@class='semi-button-content']"))).click();
            WebElement qrCodeEle = new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[@class='qrcode-image']/img[1]")));
            return qrCodeEle.getAttribute("src");// 返回对象src属性对应的值
        } catch (Exception e) {
            log.error("获取抖音登录二维码发生错误", e);
            if (Objects.nonNull(driver)) {
                this.exitBrowser(accountNo, null);
            }
            throw new BusinessException(e.getMessage());
        }
    }

    @Override
    public boolean doLogin(String accountNo) throws BusinessException {
        WebDriver driver = DRIVER_MAP.get(accountNo);
        if (Objects.isNull(driver)) {
            throw new BusinessException("登陆校验失败，请刷新二维码后重新扫码");
        }
        final WebElement element;
        try {
            element = new WebDriverWait(driver, 10, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[text()='抖音号：']/span")));
        } catch (Exception e) {
            this.exitBrowser(accountNo, null);
            throw new BusinessException("网络异常，或该人员暂未扫码登录");
        }
        if (!Objects.equals(driver.getCurrentUrl(), "https://creator.douyin.com/")) {
            if (Objects.nonNull(element)) {
                if (accountNo.equals(element.getText())) {
                    final List<WebElement> elements = new WebDriverWait(driver, 2, 300).until(driver1 ->
                            driver1.findElements(By.xpath("//span[text()='立即解锁']")));
                    if (elements.size() > 0) {
                        this.exitBrowser(accountNo, null);
                        throw new BusinessException("请联系抖音账号为: " + accountNo + " 的员工登录'抖音创作服务平台'解锁");
                    } else {
                        Integer type = this.getType().getValue();
                        common.saveCookie(driver, accountNo, type);// 保存该用户的cookies
                        accountService.updateAccountCookiesStatus(accountNo, type, true);
                        common.whenLoginCheckAccountExist(accountNo, type);
                        this.exitBrowser(accountNo, null);
                        return true;
                    }
                } else {
                    this.exitBrowser(accountNo, null);
                    throw new BusinessException("实际扫码人员与指定扫码人员不同");
                }
            }
        }
        throw new BusinessException("登陆校验失败，请刷新二维码后重新扫码");
    }

    /**
     * 获取抖音'全部作品'信息
     *
     * @param accountNo 账户号
     */
    @Override
    @Transactional
    /*public List<VideoPool> getAllVideoMsg(String accountNo) {
        final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundVideo)) {
            return hasFoundVideo;
        }
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/content/manage";
        driver.get(targetUrl);
        List<WebElement> endNodes;
        driver.manage().timeouts().implicitlyWait(1000, TimeUnit.MILLISECONDS);// 隐式等待, 定位对象时给 10s 的时间, 如果 10s 内还定位不到则抛出异常
        do {
            endNodes = driver.findElements(By.xpath("//div[starts-with(@class,'load-more')]/div[text()='没有更多视频']"));
            if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
                this.exitBrowser(accountNo, uuid);
                return null;
            }
            if (CollectionUtils.isEmpty(endNodes)) {
                //将页面滚动条拖到底部
                ((JavascriptExecutor) driver).executeScript("window.scrollTo(0,document.body.scrollHeight)");
            }
        } while (CollectionUtils.isEmpty(endNodes));
        List<WebElement> elements = driver.findElements(By.xpath("//div[@class='semi-tabs-pane-motion-overlay']//div[starts-with(@class, 'block-title')]"));
        for (WebElement element : elements) {
            String text = element.getText();
            System.out.print(text + ": ");
            List<WebElement> valueEle = element.findElements(By.xpath("following-sibling::div/div[starts-with(@class, 'block-value')]"));
            if (!CollectionUtils.isEmpty(valueEle)) {
                String value = valueEle.get(0).getText();
                System.out.println(value + "; ");
            }
        }

        // 保存'我的作品'信息
        final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
        ArrayList<VideoPool> videoPoolList = new ArrayList<>();
        String dataUrl = "https://creator.douyin.com/web/api/media/aweme/post";// 数据接口地址
        responseReceivedEvents.forEach(item -> {
            try {
                final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
                if (Objects.nonNull(response)) {
                    JSONArray jsonArray = Optional.ofNullable(JSON.parseObject(EntityUtils.toString(response.getEntity())))
                            .map(p -> p.getJSONObject("value"))
                            .map(r -> r.getJSONObject("body"))
                            .map(a -> a.getJSONArray("aweme_list"))
                            .orElse(new JSONArray());
                    jsonArray.forEach(item1 -> {
                        final JSONObject jsonObj = (JSONObject) item1;
                        final JSONObject status = jsonObj.getJSONObject("status");
                        if (Boolean.TRUE.equals(status.getBoolean("is_private"))) {
                            return;// 跳过'私有视频'
                        }
                        final JSONObject statistics = jsonObj.getJSONObject("statistics");// 获取视频'统计信息'
                        final JSONObject videoMsg = jsonObj.getJSONObject("video");// 获取'视频信息'
                        String videoId = jsonObj.getString("aweme_id");// 获取'视频id'
                        String playUrl = videoMsg.getJSONObject("play_addr")
                                .getJSONArray("url_list")
                                .stream()
                                .map(Object::toString)
                                .filter(item2 -> item2.contains("creator.douyin.com"))
                                .findFirst()
                                .orElse("");// 获取'视频播放地址'
                        if (StringUtils.hasText(playUrl)) {
                            final String[] strings = playUrl.split("&");
                            if (strings.length > 0) {
                                playUrl = strings[0];
                                try {
                                    videoId = strings[0].split("=")[1];
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }
                            }
                        }
                        videoPoolList.add(VideoPool.builder()
                                .commentCount(statistics.getInteger("comment_count"))
                                //.fullPlayCount(statistics.getInteger("play_count"))
                                .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
                                .duration(jsonObj.getLong("duration") / (1000 * 1.0))
                                .likeCount(statistics.getInteger("share_count"))
                                .accountNo(accountNo)
                                .playCount(statistics.getInteger("play_count"))
                                .reportDate(new Date())
                                .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
                                .shareCount(statistics.getInteger("share_count"))
                                .title(jsonObj.getString("desc"))
                                .type(this.getType().getValue())
                                .publishTime(new Date(jsonObj.getLong("create_time") * 1000))
                                .videoUrl(playUrl)
                                .videoId(videoId)
                                .build()
                        );
                    });
                }
            } catch (Exception e) {
                log.error("DouyinCrawl->getAllVideoMsg()发生异常", e);
            }
        });
        try {
            if (videoPoolList.size() > 0) {
                videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
            } else {
                videoPoolDao.save(VideoPool.builder()
                        .accountNo(accountNo)
                        .type(this.getType().getValue())
                        .resourceType(ResourceTypeEnum.VIDEO.getValue())
                        .reportDate(new Date())
                        .build());
            }
            log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息");
        } catch (Exception e) {
            assert log != null;
            log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败", e);
        }
        this.exitBrowser(accountNo, uuid);
        return videoPoolList;
    }*/
    public List<VideoPool> getAllVideoMsg(String accountNo) {
        ArrayList<VideoPool> videoPoolList = new ArrayList<>();
        final List<VideoPool> hasFoundVideo = common.getHasFoundVideo(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundVideo)) {
            return hasFoundVideo;
        }
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/data/stats/video";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
            this.exitBrowser(accountNo, uuid);
            return null;
        }
        this.waitFor(WAIT_SECONDS);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        WebElement element;
        try {
            element = new WebDriverWait(driver, 15, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//th[text()='作品名称'] | //span[text()='最近30天没有发布过作品'] | //span[text()='没有更多作品'] | //div/span/p")));// 获取'作品名称'th元素
        } catch (TimeoutException e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("网络阻塞, 系统稍后自动重试");
        }
        if (element.getText().contains("没有") || element.getText().contains("起开始展示数据")) {
            this.exitBrowser(accountNo, uuid);
            videoPoolDao.save(VideoPool.builder()
                    .accountNo(accountNo)
                    .type(this.getType().getValue())
                    .resourceType(ResourceTypeEnum.VIDEO.getValue())
                    .reportDate(new Date())
                    .build());
            return videoPoolList;
        } else if (element.getText().contains("作品名称")) {
            try {
                new WebDriverWait(driver, 60, 300).until(driver1 ->
                        driver1.findElements(By.xpath("//tbody/tr[@class='semi-table-row']")).size() > 0);
            } catch (TimeoutException e) {
                this.exitBrowser(accountNo, uuid);
                throw new BusinessException("网络阻塞, 系统稍后自动重试");
            }
        }
        this.waitFor(WAIT_SECONDS);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        // 保存'我的作品'信息
        try {
            final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
            String dataUrl = "https://creator.douyin.com/web/api/creator/data/item/summarize";// 数据接口地址
            responseReceivedEvents.forEach(item -> {
                try {
                    final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
                    if (Objects.nonNull(response)) {
                        JSONArray jsonArray = Optional.ofNullable(JSON.parseObject(EntityUtils.toString(response.getEntity())))
                                .map(p -> p.getJSONObject("value"))
                                .map(r -> r.getJSONObject("body"))
                                .map(a -> a.getJSONArray("item_list"))
                                .orElse(new JSONArray());
                        log.info(String.format("%s [%s]平台账户号为: %s的视频数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(jsonArray)));
                        jsonArray.forEach(item1 -> {
                            final JSONObject jsonObj = (JSONObject) item1;
                            /*final JSONObject status = jsonObj.getJSONObject("status");
                            if (Boolean.TRUE.equals(status.getBoolean("is_private"))) {
                                return;// 跳过'私有视频'
                            }*/
                            final JSONObject summarize_data = jsonObj.getJSONObject("summarize_data");
                            final JSONObject statistics = jsonObj.getJSONObject("statistics");// 获取视频'统计信息'
                            final JSONObject videoMsg = jsonObj.getJSONObject("video");// 获取'视频信息'
                            String videoId = jsonObj.getString("aweme_id");// 获取'视频id'
                            String playUrl = videoMsg.getJSONObject("play_addr")
                                    .getJSONArray("url_list")
                                    .stream()
                                    .map(Object::toString)
                                    .filter(item2 -> item2.contains("creator.douyin.com"))
                                    .findFirst()
                                    .orElse("");// 获取'视频播放地址'
                            if (StringUtils.hasText(playUrl)) {
                                final String[] strings = playUrl.split("&");
                                if (strings.length > 0) {
                                    playUrl = strings[0];
                                    try {
                                        videoId = strings[0].split("=")[1];
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                    }
                                }
                            }
                            int fullPlayCount;
                            try {
                                final BigDecimal[] bigDecimals = Optional.ofNullable(statistics.getBigDecimal("play_count")).orElse(BigDecimal.ZERO)
                                        .multiply(Optional.ofNullable(summarize_data.getBigDecimal("play_finish_ratio")).orElse(BigDecimal.ZERO))
                                        .divideAndRemainder(BigDecimal.ONE);
                                fullPlayCount = bigDecimals[0].intValue();
                                if (bigDecimals[1].compareTo(BigDecimal.ZERO) > 0) {
                                    fullPlayCount++;
                                }
                            } catch (Exception e) {
                                log.error("[抖音]计算[fullPlayCount]指标发生错误", e);
                                fullPlayCount = 0;
                            }
                            assert statistics != null;
                            assert summarize_data != null;
                            videoPoolList.add(VideoPool.builder()
                                    .commentCount(Optional.ofNullable(statistics.getInteger("comment_count")).orElse(0))
                                    .fullPlayCount(fullPlayCount)
                                    .preview(videoMsg.getJSONObject("origin_cover").getJSONArray("url_list").get(0).toString())
                                    .duration(Optional.ofNullable(jsonObj.getDouble("duration")).orElse(0d) / 1000)
                                    .likeCount(Optional.ofNullable(statistics.getInteger("digg_count")).orElse(0))
                                    .accountNo(accountNo)
                                    .playCount(Optional.ofNullable(statistics.getInteger("play_count")).orElse(0))
                                    .reportDate(new Date())
                                    .resourceType(jsonObj.getBoolean("is_live_replay") ? 2 : 1)
                                    .shareCount(Optional.ofNullable(statistics.getInteger("share_count")).orElse(0))
                                    .title(Optional.ofNullable(jsonObj.getString("desc")).orElse(""))
                                    .type(this.getType().getValue())
                                    .publishTime(new Date(Optional.ofNullable(jsonObj.getLong("create_time")).orElse(new Date().getTime()) * 1000))
                                    .newFansUserCnt(Optional.ofNullable(summarize_data.getInteger("new_fans_count")).orElse(0))
                                    .videoUrl(playUrl)
                                    .videoId(videoId)
                                    .build()
                            );
                        });
                    }
                } catch (Exception e) {
                    this.exitBrowser(accountNo, uuid);
                    assert log != null;
                    log.error("内部try-catch, DouyinCrawl->getAllVideoMsg()发生异常", e);
                    throw new BusinessException("内部try-catch, DouyinCrawl->getAllVideoMsg()发生异常");
                }
            });
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            log.error("外部try-catch, DouyinCrawl->getAllVideoMsg()发生异常", e);
            throw new BusinessException("外部try-catch, DouyinCrawl->getAllVideoMsg()发生异常");
        }
        try {
            if (videoPoolList.size() > 0) {
                videoPoolDao.saveAll(videoPoolList);// 将收集到的视频信息保存
            } else {
                videoPoolDao.save(VideoPool.builder()
                        .accountNo(accountNo)
                        .type(this.getType().getValue())
                        .resourceType(ResourceTypeEnum.VIDEO.getValue())
                        .reportDate(new Date())
                        .build());
            }
            this.exitBrowser(accountNo, uuid);
            log.info(LocalDate.now() + " 总共获取到" + videoPoolList.size() + "条抖音视频信息");
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            assert log != null;
            log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败", e);
            throw new BusinessException(LocalDate.now() + " 执行保存" + accountNo + "的抖音视频数据失败");
        }
        return videoPoolList;
    }

    /**
     * 获取昨天'直播'数据总览
     *
     * @param accountNo 账户号
     */
    public LiveOverview getYesterdayLiveOverview(String accountNo) {
        AtomicReference<LiveOverview> liveOverview = new AtomicReference<>(new LiveOverview());
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/data/live/overview";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("登录已过期, 请重新登录");
        }
        new WebDriverWait(driver, 10, 300).until((ExpectedCondition<Boolean>) driver1 ->
        {
            assert driver1 != null;
            return driver1.findElements(By.xpath("//span[text()='昨日关键数据']")).size() != 0;
        });
        this.waitFor(WAIT_SECONDS * 2);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        // 保存昨天'直播'数据总览
        final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
        String dataUrl = "https://creator.douyin.com/aweme/v1/creator/data/live/anchor/summarize";// 数据接口地址
        responseReceivedEvents.forEach(item -> {
            JSONObject summarizeData;
            try {
                final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
                if (Objects.nonNull(response)) {
                    summarizeData = JSON.parseObject(EntityUtils.toString(response.getEntity()))
                            .getJSONObject("value")
                            .getJSONObject("body")
                            .getJSONObject("summarize_data");
                    liveOverview.set(LiveOverview.builder()
                            .type(this.getType().getValue())
                            .reportDate(new Date())
                            .uploadTime(new Date())
                            .accountNo(accountNo)
                            .liveCnt(summarizeData.getInteger("yes_live_cnt"))
                            .liveDuration(summarizeData.getDouble("yes_live_duration"))
                            .watchTimesCnt(summarizeData.getInteger("yes_watch_cnt"))
                            .watchUserCnt(summarizeData.getInteger("yes_watch_ucnt"))
                            .maxWatchUserCnt(summarizeData.getInteger("yes_watch_pcu"))
                            .watchAvgDuration(summarizeData.getDouble("yes_watch_avg_time"))
                            .commentCnt(summarizeData.getInteger("yes_comment_cnt"))
                            .commentUserCnt(summarizeData.getInteger("yes_comment_ucnt"))
                            .newFansCnt(summarizeData.getInteger("yes_new_fans_ucnt"))
                            .consumeUserCnt(summarizeData.getInteger("yes_consume_ucnt"))
                            .giftCnt(summarizeData.getInteger("yes_gift_cnt"))
                            .score(summarizeData.getInteger("yes_score"))
                            .income(summarizeData.getDouble("yes_income"))
                            .build());
                    liveDataDao.save(liveOverview.get());
                }
            } catch (Exception e) {
                log.error("DouyinCrawl->getYesterdayLiveOverview()发生异常", e);
            }
        });
        this.exitBrowser(accountNo, uuid);
        return liveOverview.get();
    }

    /**
     * 保存昨日所有直播详情数据
     *
     * @param accountNo
     */
    @Override
    @Transactional
    public List<LivePool> getYesterdayLiveMsg(String accountNo) {
        List<LivePool> livePoolList = new ArrayList<>();
        final List<LivePool> hasFoundLive = common.getHasFoundLive(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundLive)) {
            return hasFoundLive;
        }
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/data/live/video";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
            this.exitBrowser(accountNo, uuid);
            return null;
        }
        this.waitFor(WAIT_SECONDS);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        WebElement element;
        try {
            element = new WebDriverWait(driver, 15, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//th[text()='直播封面/名称'] | //span[starts-with(@class,'empty')][text()='暂无数据']")));
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("爬取'抖音'直播数据时, 网络阻塞, 或者页面加载异常");
        }
        if (element.getText().contains("无数据")) {
            this.exitBrowser(accountNo, uuid);
            log.info(LocalDate.now() + " 暂未找到账户号为:" + accountNo + "的抖音直播数据");
            final LivePool nullLive = LivePool.builder()
                    .type(this.getType().getValue())
                    .accountNo(accountNo)
                    .reportDate(new Date())
                    .build();
            livePoolDao.save(nullLive);
            //livePoolList.add(nullLive);
            return livePoolList;
        } else if (element.getText().contains("直播封面")) {
            try {
                new WebDriverWait(driver, 60, 300).until(driver1 ->
                        driver1.findElements(By.xpath("//tbody/tr[@class='semi-table-row']")).size() > 0);
            } catch (TimeoutException e) {
                this.exitBrowser(accountNo, uuid);
                throw new BusinessException("网络阻塞, 系统稍后自动重试");
            }
        }
        this.waitFor(WAIT_SECONDS);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        // 读http日志, 处理数据
        try {
            final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
            String dataUrl = "https://creator.douyin.com/aweme/v1/creator/data/live/room/list";// 数据接口地址
            responseReceivedEvents.forEach(item -> {
                JSONArray roomList;
                try {
                    final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
                    if (Objects.nonNull(response)) {
                        roomList = JSON.parseObject(EntityUtils.toString(response.getEntity()))
                                .getJSONObject("value")
                                .getJSONObject("body")
                                .getJSONArray("room_list");
                        log.info(String.format("%s [%s]平台账户号为: %s的直播数据的原始数据为: %s", LocalDateTime.now(), this.getType().getName(), accountNo, JSON.toJSONString(roomList)));
                        if (roomList.size() > 0) {
                            roomList.forEach(item1 -> {
                                final JSONObject obj = (JSONObject) item1;
                                final Date open_time = Optional.ofNullable(obj.getDate("open_time")).orElse(new Date());
                                final Date yesterday = DateUtil.getThisDayMinTime(DateUtil.getPreviousDay(new Date()));
                                if (open_time.after(yesterday)) {
                                    /**
                                     * -"comment_ucnt":"3",
                                     * -"consume_ucnt":"0",
                                     * -"duration":"81.4",
                                     * -"income":"0.00",
                                     * -"new_fans_ucnt":"1",
                                     * -"open_time":"2021-12-13 10:53",
                                     * -"room_cover_image":"http://p3-webcast.douyinpic.com/img/tos-cn-i-0813/6a649b8b6ab343e19e1cde06c0f766cf~tplv-obj.image",
                                     * -"room_id":"7041014325120731939",
                                     * -"room_name":"长安汽车  年终盛典",
                                     * -"score":"0",
                                     * -"watch_cnt":"56",
                                     * -"watch_pcu":"8",
                                     * -"watch_ucnt":"18"
                                     */
                                    livePoolList.add(LivePool.builder()
                                            .type(this.getType().getValue())
                                            .accountNo(accountNo)
                                            .reportDate(new Date())
                                            .commentUserCnt(Optional.ofNullable(obj.getInteger("comment_ucnt")).orElse(0))// 评论人数
                                            .consumeUserCnt(Optional.ofNullable(obj.getInteger("consume_ucnt")).orElse(0))// 付费人数
                                            .duration(Optional.ofNullable(obj.getDouble("duration")).orElse(0d))// 直播时长
                                            .income(Optional.ofNullable(obj.getDouble("income")).orElse(0d))// 直播收益(元)
                                            .newFansUserCnt(Optional.ofNullable(obj.getInteger("new_fans_ucnt")).orElse(0))// 新增粉丝
                                            .openTime(open_time)// 开播时间
                                            .roomCoverImage(obj.getString("room_cover_image"))// 直播间封面图
                                            .roomId(Optional.ofNullable(obj.getString("room_id")).orElse(""))// 直播间id
                                            .roomName(Optional.ofNullable(obj.getString("room_name")).orElse(""))// 直播封面/名称
                                            .score(Optional.ofNullable(obj.getInteger("score")).orElse(0))// 收获音浪
                                            .watchCnt(Optional.ofNullable(obj.getInteger("watch_cnt")).orElse(0))// 观看人次
                                            .watchPeakUserCnt(Optional.ofNullable(obj.getInteger("watch_pcu")).orElse(0))// 在线人数巅峰
                                            .watchUserCnt(Optional.ofNullable(obj.getInteger("watch_ucnt")).orElse(0))// 观众人数
                                            .getPlaybackFailTimes(0)
                                            .build()
                                    );
                                }
                            });
                        }
                    }
                } catch (Exception e) {
                    this.exitBrowser(accountNo, uuid);
                    assert log != null;
                    log.error("DouyinCrawl->getYesterdayLiveMsg()发生异常", e);
                    throw new BusinessException("内部try-catch, DouyinCrawl->getYesterdayLiveMsg()发生异常");
                }
            });
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            log.error("DouyinCrawl->getYesterdayLiveMsg()发生异常", e);
            throw new BusinessException("外部try-catch, DouyinCrawl->getYesterdayLiveMsg()发生异常");
        }
        Date previousDay = DateUtil.getPreviousDay(new Date());
        Date endTime = DateUtil.getThisDayMaxTime(previousDay);
        Date startTime = DateUtil.getThisDayMinTime(previousDay);
        try {
            if (livePoolList.size() > 0) {
                livePoolDao.saveAll(livePoolList.stream()
                        .filter(item -> item.getOpenTime().compareTo(startTime) >= 0 && item.getOpenTime().compareTo(endTime) <= 0)
                        .collect(Collectors.toList())
                );
            } else {
                livePoolDao.save(LivePool.builder()
                        .type(this.getType().getValue())
                        .accountNo(accountNo)
                        .reportDate(new Date())
                        .build());
            }
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            log.error(LocalDate.now() + " 执行保存" + accountNo + "的抖音直播数据失败", e);
            throw new BusinessException(LocalDate.now() + " 执行保存" + accountNo + "的抖音直播数据失败");
        }
        this.exitBrowser(accountNo, uuid);
        return livePoolList;
    }

    /**
     * 更新用户粉丝数
     *
     * @param accountNo
     */
    @Override
    @Transactional
    public ReportAccountDto updateAccountMsg(String accountNo) {
        final ReportAccountDto hasFoundAccountMsg = common.getHasFoundAccountMsg(accountNo, this.getType().getValue(), DateUtil.getThisDayMinTime(new Date()));
        if (Objects.nonNull(hasFoundAccountMsg)) {
            return hasFoundAccountMsg;
        }
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/home";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        if (!Objects.equals(driver.getCurrentUrl(), targetUrl)) {
            this.exitBrowser(accountNo, uuid);
            return null;
        }
        this.waitFor(WAIT_SECONDS);// 线程休眠WAIT_SECONDS指定的秒, 等待web页面加载数据
        try {
            new WebDriverWait(driver, 15, 300).until(driver1 ->
                    driver1.findElement(By.xpath("//div[text()='粉丝'] | //div[starts-with(@class,'signature')]")));
        } catch (TimeoutException e) {
            this.exitBrowser(accountNo, uuid);
            log.info("网络阻塞");
            throw new BusinessException(e.getMessage());
        }
        /*try {
            WebElement startExperienceButton = new WebDriverWait(driver, 6, 2000).until(driver1 ->
                    driver1.findElement(By.xpath("//div[@class='semi-modal-body']/button[starts-with(@class,'semi-button') and @type='button']")));
            if (Objects.nonNull(startExperienceButton)) {
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-modal-body']/button[starts-with(@class,'semi-button') and @type='button']"))).click();
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-popover']//button[starts-with(@class,'semi-button') and @type='button']"))).click();
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-popover']//button[starts-with(@class,'semi-button') and @type='button']"))).click();
            }
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException(e.getMessage());
        }*/
        // 读http日志, 处理数据
        final List<ResponseReceived> responseReceivedEvents = common.processHttpTransferData(driver);
        String dataUrl = "https://creator.douyin.com/web/api/media/user/info";// 数据接口地址
        AtomicReference<ReportAccountDto> reportAccountDto = new AtomicReference<>();
        responseReceivedEvents.forEach(item -> {
            JSONObject data;
            try {
                final HttpResponse response = common.getHttpResponse(driver, item, dataUrl);
                if (Objects.nonNull(response)) {
                    final JSONObject body = JSON.parseObject(EntityUtils.toString(response.getEntity()))
                            .getJSONObject("value")
                            .getJSONObject("body");
                    if (Objects.isNull(body)) {
                        return;
                    }
                    data = body.getJSONObject("user");
                    if (Objects.isNull(data)) {
                        return;
                    }
                    reportAccountDto.set(ReportAccountDto.builder()
                            .fansCnt(data.getInteger("follower_count"))
                            .followCnt(data.getInteger("following_count"))
                            .likeCnt(data.getInteger("total_favorited"))
                            .accountId(data.getString("unique_id"))
                            .userOtherId(data.getString("uid"))
                            .accountName(data.getString("nickname"))
                            .build()
                    );
                    final Account account = accountDao.findByAccountNoAndType(accountNo, this.getType().getValue());
                    if (Objects.nonNull(account))
                        accountDao.updateMsg(account.getId(), reportAccountDto.get().getAccountName(), reportAccountDto.get().getFansCnt(), new Date());
                    else
                        accountDao.save(Account.builder()
                                .cookiesStatus(true)
                                .accountNo(accountNo)
                                .type(this.getType().getValue())
                                .fansCnt(reportAccountDto.get().getFansCnt())
                                .accountName(reportAccountDto.get().getAccountName())
                                .reportDate(new Date())
                                .done(false)
                                .build());
                }
            } catch (Exception e) {
                this.exitBrowser(accountNo, uuid);
                log.error("DouyinCrawl->updateAccountFans()发生异常", e);
                throw new BusinessException("DouyinCrawl->updateAccountFans()发生异常");
            }
        });
        this.exitBrowser(accountNo, uuid);
        return reportAccountDto.get();
    }

    /**
     * 进入指定账户号的创作者平台首页
     *
     * @param accountNo 抖音账户号
     */
    public void loginIndex(String accountNo) {
        final String uuid = UUID.randomUUID().toString().replace("-", "");
        final WebDriver driver = this.getDYDriver(accountNo, uuid);
        String targetUrl = "https://creator.douyin.com/creator-micro/home";
        try {
            driver.get(targetUrl);
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException("跳转页面发生异常");
        }
        try {
            WebElement startExperienceButton = new WebDriverWait(driver, 6, 2000).until(driver1 ->
                    driver1.findElement(By.xpath("//div[@class='semi-modal-body']/button[starts-with(@class,'semi-button') and @type='button']")));
            if (Objects.nonNull(startExperienceButton)) {
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-modal-body']/button[starts-with(@class,'semi-button') and @type='button']"))).click();
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-popover']//button[starts-with(@class,'semi-button') and @type='button']"))).click();
                new WebDriverWait(driver, 6, 500).until(driver1 ->
                        driver1.findElement(By.xpath("//div[@class='semi-popover']//button[starts-with(@class,'semi-button') and @type='button']"))).click();
            }
        } catch (Exception e) {
            this.exitBrowser(accountNo, uuid);
            throw new BusinessException(e.getMessage());
        }
        this.exitBrowser(accountNo, uuid);
    }

    /**
     * 将抖音账户的cookies放入driver中
     *
     * @param accountNo
     * @return
     */
    private WebDriver getDYDriver(String accountNo, String uuid) {
        WebDriver driver = common.createDriver();
        try {
            driver.get("https://creator.douyin.com");
            Set<Cookie> cookieSet = common.loadCookie(accountNo, this.getType().getValue())
                    .stream()
                    .map(FwCookie::seleniumCookie)
                    .collect(Collectors.toSet());
            cookieSet.forEach(item -> driver.manage().addCookie(item));// 给driver加载账户的cookies;
            DRIVER_MAP.put(accountNo + "#" + uuid, driver);
            return driver;
        } catch (Exception e) {
            driver.close();
            throw new BusinessException("创建抖音驱动发生异常: " + e.getMessage());
        }
    }

    /**
     * 退出浏览器
     *
     * @param accountNo 账户号
     * @return
     */
    public boolean exitBrowser(String accountNo, String uuid) {
        String key = accountNo + (Objects.nonNull(uuid) ? "#" + uuid : "");
        WebDriver driver = DRIVER_MAP.get(key);
        if (Objects.isNull(driver)) {
            return true;
        }
        driver.close();
        DRIVER_MAP.remove(key);
        return true;
    }
}