open-platform/src/test/java/com/zhiqiyun/SimplePageProcessor.java

package com.zhiqiyun;

import com.zhiqiyun.open.core.models.sentiment.SentimentSpiderSiteRule;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class SimplePageProcessor {
    public static void main(String[] args) {
        SentimentSpiderSiteRule rule = new SentimentSpiderSiteRule();
        rule.setStartUrls("https://www.188420.com/");
//        rule.setUrlPatterns("https://www.188420.com/a/([0-9]+).html");
//        rule.setXpath("/html/body/div[3]/div[1]/div[1]");

        String[] startUrlArray = rule.getStartUrls().split("\n");

//        Spider spider = Spider.create(new SentimentSpiderEventServiceImpl.SpiderEventProcessor(rule));
//
//
//        List<Pipeline> pipelines = new ArrayList<>();
//        pipelines.add((resultItems, task) -> {
//            Map<String, Object> dataMap = resultItems.getAll();
//
//            String url = resultItems.getRequest().getUrl();
//            String id = DigestUtils.md5Hex(url.replace("http://", "").replace("https://", ""));
//
//            String title = dataMap.getOrDefault("title", "").toString();
//            String keywords = dataMap.getOrDefault("keywords", "").toString();
//            String description = dataMap.getOrDefault("description", "").toString();
//            String bodyText = dataMap.getOrDefault("bodyText", "").toString();
//            String html = dataMap.getOrDefault("html", "").toString();
//            List<String> listFragments = JSON.parseArray(dataMap.getOrDefault("listFragments", "[]").toString(), String.class);
//
//            String[] listKeywords = new String[]{"十三届五次会", "习近平"};
//
//            boolean flag = true;
//            for (String keyword : listKeywords) {
//                if (!bodyText.contains(keyword)) {
//                    flag = false;
//                    break;
//                }
//            }
//
//            listFragments.removeIf(o -> {
//                boolean hasContains = false;
//                for (String keyword : listKeywords) {
//                    if (o.contains(keyword)) {
//                        hasContains = true;
//                        break;
//                    }
//                }
//                return !hasContains;
//            });
//
//            log.info("{}>>>>>>>>{}>>>>>>>>{}>>>>>>>>{}", url, flag, listFragments.size(), bodyText);
//        });
//        spider.setPipelines(pipelines);
//        spider.setExitWhenComplete(true);
//        spider.addUrl(startUrlArray);
//        OkHttpClient.Builder builder = new OkHttpClient.Builder();
//        builder.connectTimeout(30, TimeUnit.SECONDS);
//        spider.setDownloader(new OkHttpDownloader());
//        spider.runAsync();
    }
}