package com.zhiqiyun.open.core.spiders;

import com.alibaba.fastjson.JSON;
import com.zhiqiyun.open.core.models.sentiment.SentimentSpiderSiteRule;
import com.zhiqiyun.open.core.models.sentiment.UrlPatterns;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.CssSelector;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.selector.XpathSelector;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;

@Slf4j
public class HtmlPageProcessor implements PageProcessor {
    private final List<String> LIST_SPIDER_URLS = new ArrayList<>();

    private final Site site;

    private final SentimentSpiderSiteRule rule;


    public HtmlPageProcessor(SentimentSpiderSiteRule rule) {
        this.rule = rule;
        this.site = Site.me();
        this.site.setUseGzip(true);
        this.site.setRetryTimes(3);
        this.site.setSleepTime(1000);
    }

    @Override
    public void process(Page page) {
        List<String> listUrls = page.getHtml().links().all();

        listUrls.removeIf(url -> StringUtils.isBlank(url) || LIST_SPIDER_URLS.contains(url));

        List<UrlPatterns> listUrlPatterns = this.rule.getUrlPatterns();

        AtomicReference<UrlPatterns> matchedUrlPattern = new AtomicReference<>(null);
        if (listUrlPatterns != null) {
            listUrls.removeIf(url -> {
                log.info(url);
                boolean hasMatched = false;
                for (UrlPatterns urlPattern : listUrlPatterns) {
                    hasMatched = Pattern.matches(urlPattern.getUrlPattern(), url);
                    if (hasMatched) {
                        break;
                    }
                }
                return !hasMatched;
            });

            for (UrlPatterns urlPattern : listUrlPatterns) {
                if (Pattern.matches(urlPattern.getUrlPattern(), page.getUrl().get())) {
                    matchedUrlPattern.set(urlPattern);
                }
            }
        }

        LIST_SPIDER_URLS.addAll(listUrls);

        page.addTargetRequests(listUrls);

        Document document = page.getHtml().getDocument();

        String title = document.title();
        String bodyText = "";
        String html = document.html();

        String keywords = "";
        String description = "";
        Elements elements = document.getElementsByTag("meta");

        if (elements != null && elements.size() > 0) {
            for (Element element : elements) {
                String metaName = element.attr("name");
                String metaContent = element.attr("content");
                if (StringUtils.equalsIgnoreCase("keywords", metaName)) {
                    keywords = metaContent;
                }
                if (StringUtils.equalsIgnoreCase("description", metaName)) {
                    description = metaContent;
                }
            }
        }
        Selectable selectable;
        UrlPatterns urlPatterns = null;
        if (matchedUrlPattern.get() != null) {
            urlPatterns = matchedUrlPattern.get();
        }

        if (urlPatterns == null) {
            selectable = page.getHtml();
        } else if ("XpathSelector".equals(urlPatterns.getSelectorName())) {
            XpathSelector xpathSelector = new XpathSelector(urlPatterns.getSelectorText());
            selectable = page.getHtml().select(xpathSelector);
        } else if ("CssSelector".equals(urlPatterns.getSelectorName())) {
            CssSelector cssSelector = new CssSelector(urlPatterns.getSelectorText());
            selectable = page.getHtml().select(cssSelector);
        } else {
            selectable = page.getHtml();
        }

        List<String> listFragments = new ArrayList<>();
        if (selectable.match()) {
            document.html(selectable.get());
            bodyText = document.text();

            Elements allElements = document.children().first().children();
            for (Element element : allElements) {
                String text = element.text();
                if (StringUtils.isNotBlank(text)) {
                    listFragments.add(text.trim());
                }
            }
        }

        log.info("{}>>>>>>>>{}>>>>>>>>{}", page.getUrl(), listFragments.size(), bodyText);

        if (StringUtils.isBlank(bodyText)) {
            page.setSkip(true);
        }


        if (StringUtils.isBlank(description) && StringUtils.isNotBlank(bodyText)) {
            description = bodyText.length() >= 200 ? bodyText.substring(0, 200) : bodyText;
        }

        if (StringUtils.isBlank(keywords) && StringUtils.isNotBlank(description)) {
            keywords = description;
        }

        page.putField("title", title);
        page.putField("bodyText", bodyText);
        page.putField("html", html);
        page.putField("keywords", keywords);
        page.putField("description", description);
        page.putField("listFragments", JSON.toJSONString(listFragments));
    }

    @Override
    public Site getSite() {
        return this.site;
    }
}
