update

src/main/java/com/zhiqiyun/open/core/models/statistics/PopularFeelingsPage.java

@@ -1,5 +1,7 @@
 package com.zhiqiyun.open.core.models.statistics;
 
+import com.baomidou.mybatisplus.annotation.FieldStrategy;
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
 
@@ -15,7 +17,11 @@ public class PopularFeelingsPage {
     private Long popularFeelingsId;
     private String url;
     private String title;
+    private String keywords;
+    private String description;
     private String bodyText;
     private String html;
+    @TableField(updateStrategy = FieldStrategy.NEVER)
     private Date spiderTime;
+    private Date updateTime;
 }

src/main/java/com/zhiqiyun/open/core/service/impl/PopularFeelingsServiceImpl.java

@@ -2,21 +2,26 @@ package com.zhiqiyun.open.core.service.impl;
 
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.zhiqiyun.open.core.mapper.statistics.PopularFeelingsMapper;
-import com.zhiqiyun.open.core.mapper.statistics.PopularFeelingsPageMapper;
 import com.zhiqiyun.open.core.models.statistics.PopularFeelings;
 import com.zhiqiyun.open.core.models.statistics.PopularFeelingsPage;
+import com.zhiqiyun.open.core.service.PopularFeelingsPageService;
 import com.zhiqiyun.open.core.service.PopularFeelingsService;
 import com.zhiqiyun.open.utils.DateUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import us.codecraft.webmagic.*;
+import us.codecraft.webmagic.pipeline.Pipeline;
 import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author jtoms
@@ -26,13 +31,13 @@ import java.util.List;
 public class PopularFeelingsServiceImpl extends ServiceImpl<PopularFeelingsMapper, PopularFeelings> implements PopularFeelingsService {
 
 	@Autowired
-	private PopularFeelingsPageMapper popularFeelingsPageMapper;
+	private PopularFeelingsPageService popularFeelingsPageService;
 
 	@Override
 	public void start(PopularFeelings popular) {
 		String[] urls = popular.getStartUrls().toArray(new String[]{});
 
-		Spider spider = Spider.create(new PopularFeelingsProcessor(popular, this.popularFeelingsPageMapper));
+		Spider spider = Spider.create(new PopularFeelingsProcessor(popular));
 		List<SpiderListener> listListeners = new ArrayList<>();
 		listListeners.add(new SpiderListener() {
 			@Override
@@ -43,9 +48,43 @@ public class PopularFeelingsServiceImpl extends ServiceImpl<PopularFeelingsMappe
 			@Override
 			public void onError(Request request) {
 				log.info("onError>>>>>>{}", request.getUrl());
-
 			}
 		});
+		List<Pipeline> pipelines = new ArrayList<>();
+		pipelines.add(new Pipeline() {
+			@Override
+			public void process(ResultItems resultItems, Task task) {
+				Map<String, Object> dataMap = resultItems.getAll();
+				log.info(resultItems.getRequest().getUrl());
+				String url = resultItems.getRequest().getUrl();
+				String id = DigestUtils.md5Hex(url.replace("http://", "").replace("https://", ""));
+
+				String title = dataMap.getOrDefault("title", "").toString();
+				String keywords = dataMap.getOrDefault("keywords", "").toString();
+				String description = dataMap.getOrDefault("description", "").toString();
+				String bodyText = dataMap.getOrDefault("bodyText", "").toString();
+				String html = dataMap.getOrDefault("html", "").toString();
+
+
+				if (html.contains(popular.getKeywords())) {
+					PopularFeelingsPage popularFeelingsPage = new PopularFeelingsPage();
+
+					popularFeelingsPage.setId(id);
+					popularFeelingsPage.setPopularFeelingsId(popular.getId());
+					popularFeelingsPage.setUrl(url);
+					popularFeelingsPage.setTitle(title);
+					popularFeelingsPage.setKeywords(keywords);
+					popularFeelingsPage.setDescription(description);
+					popularFeelingsPage.setBodyText(bodyText);
+					popularFeelingsPage.setHtml(html);
+					popularFeelingsPage.setSpiderTime(DateUtil.current());
+					popularFeelingsPage.setUpdateTime(DateUtil.current());
+					popularFeelingsPageService.saveOrUpdate(popularFeelingsPage);
+				}
+			}
+		});
+		spider.setPipelines(pipelines);
+		spider.setExitWhenComplete(true);
 		spider.setSpiderListeners(listListeners);
 		spider.addUrl(urls);
 		spider.runAsync();
@@ -61,45 +100,59 @@ public class PopularFeelingsServiceImpl extends ServiceImpl<PopularFeelingsMappe
 
 		private final Site site;
 
-		private final PopularFeelings popular;
 
-		private final PopularFeelingsPageMapper popularFeelingsPageMapper;
-
-		public PopularFeelingsProcessor(PopularFeelings popular, PopularFeelingsPageMapper popularFeelingsPageMapper) {
-			this.popular = popular;
-			this.site = Site.me().setDomain(this.popular.getDomain()).setRetryTimes(3).setSleepTime(1000);
-			this.popularFeelingsPageMapper = popularFeelingsPageMapper;
+		public PopularFeelingsProcessor(PopularFeelings popular) {
+			this.site = Site.me();
+			this.site.setDomain(popular.getDomain());
+			this.site.setUseGzip(true);
+			this.site.setRetryTimes(3);
+			this.site.setSleepTime(1000);
 		}
 
 		public void process(Page page) {
 			List<String> listUrls = page.getHtml().links().all();
-			listUrls.removeIf(s -> {
-				log.info(s);
-				return LIST_SPIDER_URLS.contains(s) || StringUtils.isBlank(s);
-			});
+			listUrls.removeIf(s -> StringUtils.isBlank(s) || LIST_SPIDER_URLS.contains(s));
 			LIST_SPIDER_URLS.addAll(listUrls);
-			log.info(">>>>>>>>{}", LIST_SPIDER_URLS.size());
+
 			page.addTargetRequests(listUrls);
 
 			Document document = page.getHtml().getDocument();
-			String uri = document.baseUri();
-			log.info(uri);
+
 			String title = document.title();
 			String bodyText = document.body().text();
 			String html = document.html();
-			log.info("{}>>>>>>{}", uri, title);
-			if (title.contains(popular.getKeywords()) || bodyText.contains(popular.getKeywords())) {
-				PopularFeelingsPage popularFeelingsPage = new PopularFeelingsPage();
-				popularFeelingsPage.setId(document.id());
-				popularFeelingsPage.setPopularFeelingsId(popular.getId());
-				popularFeelingsPage.setUrl(uri);
-				popularFeelingsPage.setTitle(title);
-				popularFeelingsPage.setBodyText(bodyText);
-				popularFeelingsPage.setHtml(html);
-				popularFeelingsPage.setSpiderTime(DateUtil.current());
-				this.popularFeelingsPageMapper.insert(popularFeelingsPage);
+
+			String keywords = "";
+			String description = "";
+			Elements elements = document.getElementsByTag("meta");
+
+			if (elements != null && elements.size() > 0) {
+				for (Element element : elements) {
+					String metaName = element.attr("name");
+					String metaContent = element.attr("content");
+					if ("keywords".equals(metaName)) {
+						keywords = metaContent;
+					}
+					if ("description".equals(metaName)) {
+						description = metaContent;
+					}
+				}
+			}
+
+			if (StringUtils.isBlank(description) && StringUtils.isNotBlank(bodyText)) {
+				description = bodyText.length() >= 200 ? bodyText.substring(0, 200) : bodyText;
 			}
 
+			if (StringUtils.isBlank(keywords) && StringUtils.isNotBlank(bodyText)) {
+				description = keywords;
+			}
+
+
+			page.putField("title", title);
+			page.putField("bodyText", bodyText);
+			page.putField("html", html);
+			page.putField("keywords", keywords);
+			page.putField("description", description);
 		}
 
 		public Site getSite() {

src/main/java/com/zhiqiyun/open/mvc/controller/PopularFeelingsController.java

@@ -35,114 +35,114 @@ import java.util.List;
 @RequestMapping("/popular/feelings")
 public class PopularFeelingsController {
 
-    @Autowired
-    private PopularFeelingsService popularFeelingsService;
+	@Autowired
+	private PopularFeelingsService popularFeelingsService;
 
 
-    @Autowired
-    private PopularFeelingsPageService popularFeelingsPageService;
+	@Autowired
+	private PopularFeelingsPageService popularFeelingsPageService;
 
-    @Autowired
-    private OauthService oauthService;
+	@Autowired
+	private OauthService oauthService;
 
-    @Autowired
-    private SequenceService sequenceService;
+	@Autowired
+	private SequenceService sequenceService;
 
 
-    @Permission(value = "popular.feelings.find", tags = "查询舆情监控")
-    @PostMapping("/findPage")
-    public Result findPage(@RequestBody QueryPopularFeelingsParam param) {
+	@Permission(value = "popular.feelings.find", tags = "查询舆情监控")
+	@PostMapping("/findPage")
+	public Result findPage(@RequestBody QueryPopularFeelingsParam param) {
 
-        QueryWrapper<PopularFeelings> wrapper = new QueryWrapper<>();
+		QueryWrapper<PopularFeelings> wrapper = new QueryWrapper<>();
 
-        if (StringUtils.isNotBlank(param.getTitle())) {
-            wrapper.like("title", param.getTitle());
-        }
+		if (StringUtils.isNotBlank(param.getTitle())) {
+			wrapper.like("title", param.getTitle());
+		}
 
-        if (StringUtils.isNotBlank(param.getKeywords())) {
-            wrapper.like("keywords", param.getKeywords());
-        }
+		if (StringUtils.isNotBlank(param.getKeywords())) {
+			wrapper.like("keywords", param.getKeywords());
+		}
 
-        if (StringUtils.isNotBlank(param.getSiteUrl())) {
-            wrapper.like("site_url", param.getSiteUrl());
-        }
+		if (StringUtils.isNotBlank(param.getSiteUrl())) {
+			wrapper.like("site_url", param.getSiteUrl());
+		}
 
-        Page<PopularFeelings> page = param.getPage();
-        page.addOrder(OrderItem.desc("id"));
+		Page<PopularFeelings> page = param.getPage();
+		page.addOrder(OrderItem.desc("id"));
 
-        Page<PopularFeelings> resultData = this.popularFeelingsService.page(page, wrapper);
+		Page<PopularFeelings> resultData = this.popularFeelingsService.page(page, wrapper);
 
-        return Result.instance(Result.Code.SUCCESS).setData(resultData);
-    }
+		return Result.instance(Result.Code.SUCCESS).setData(resultData);
+	}
 
-    @Permission(value = "popular.feelings.add", tags = "新建舆情监控")
-    @PostMapping("/save")
-    public Result save(@Valid @RequestBody SavePopularFeelingsParam param) throws Exception {
+	@Permission(value = "popular.feelings.add", tags = "新建舆情监控")
+	@PostMapping("/save")
+	public Result save(@Valid @RequestBody SavePopularFeelingsParam param) throws Exception {
 
-        PopularFeelings entity = new PopularFeelings();
-        BeanUtils.copyProperties(param, entity);
+		PopularFeelings entity = new PopularFeelings();
+		BeanUtils.copyProperties(param, entity);
 
-        OauthInfo oauthInfo = this.oauthService.getAuth(ServletContext.getAccessToken());
+		OauthInfo oauthInfo = this.oauthService.getAuth(ServletContext.getAccessToken());
 
-        entity.setCreatedTime(DateUtil.current());
-        entity.setCreatedBy(oauthInfo.getId());
+		entity.setCreatedTime(DateUtil.current());
+		entity.setCreatedBy(oauthInfo.getId());
 
-        entity.setUpdatedTime(DateUtil.current());
-        entity.setUpdatedBy(oauthInfo.getId());
-        entity.setId(this.sequenceService.nextId());
+		entity.setUpdatedTime(DateUtil.current());
+		entity.setUpdatedBy(oauthInfo.getId());
+		entity.setId(this.sequenceService.nextId());
 
-        this.popularFeelingsService.save(entity);
+		this.popularFeelingsService.save(entity);
 
-        return Result.instance(Result.Code.MESSAGE_SUCCESS);
-    }
+		return Result.instance(Result.Code.MESSAGE_SUCCESS);
+	}
 
-    @Permission(value = "popular.feelings.edit", tags = "更新舆情监控")
-    @PostMapping("/updateById")
-    public Result updateById(Long id, @Valid @RequestBody SavePopularFeelingsParam param) {
-        PopularFeelings entity = new PopularFeelings();
-        BeanUtils.copyProperties(param, entity);
+	@Permission(value = "popular.feelings.edit", tags = "更新舆情监控")
+	@PostMapping("/updateById")
+	public Result updateById(Long id, @Valid @RequestBody SavePopularFeelingsParam param) {
+		PopularFeelings entity = new PopularFeelings();
+		BeanUtils.copyProperties(param, entity);
 
-        OauthInfo oauthInfo = this.oauthService.getAuth(ServletContext.getAccessToken());
+		OauthInfo oauthInfo = this.oauthService.getAuth(ServletContext.getAccessToken());
 
-        entity.setUpdatedBy(oauthInfo.getId());
-        entity.setUpdatedTime(DateUtil.current());
-        entity.setId(id);
+		entity.setUpdatedBy(oauthInfo.getId());
+		entity.setUpdatedTime(DateUtil.current());
+		entity.setId(id);
 
-        this.popularFeelingsService.updateById(entity);
+		this.popularFeelingsService.updateById(entity);
 
-        return Result.instance(Result.Code.MESSAGE_SUCCESS);
-    }
+		return Result.instance(Result.Code.MESSAGE_SUCCESS);
+	}
 
-    @Permission(value = "popular.feelings.delete", tags = "删除舆情监控")
-    @PostMapping("/deleteByIds")
-    public Result deleteByIds(@RequestBody List<Long> ids) {
-        this.popularFeelingsService.removeByIds(ids);
-        return Result.instance(Result.Code.MESSAGE_SUCCESS);
-    }
+	@Permission(value = "popular.feelings.delete", tags = "删除舆情监控")
+	@PostMapping("/deleteByIds")
+	public Result deleteByIds(@RequestBody List<Long> ids) {
+		this.popularFeelingsService.removeByIds(ids);
+		return Result.instance(Result.Code.MESSAGE_SUCCESS);
+	}
 
-    @Permission(value = "popular.feelings.delete", tags = "启动舆情监控")
-    @PostMapping("/startSpider")
-    public Result startSpider(@RequestBody List<Long> ids) {
-        List<PopularFeelings> listData = this.popularFeelingsService.listByIds(ids);
-        for (PopularFeelings popularFeelings : listData) {
-            this.popularFeelingsService.start(popularFeelings);
-        }
-        return Result.instance(Result.Code.MESSAGE_SUCCESS);
-    }
+	@Permission(value = "popular.feelings.delete", tags = "启动舆情监控")
+	@PostMapping("/startSpider")
+	public Result startSpider(@RequestBody List<Long> ids) {
+		List<PopularFeelings> listData = this.popularFeelingsService.listByIds(ids);
+		for (PopularFeelings popularFeelings : listData) {
+			this.popularFeelingsService.start(popularFeelings);
+		}
+		return Result.instance(Result.Code.MESSAGE_SUCCESS);
+	}
 
-    @Permission(value = "popular.feelings.delete", tags = "查询舆情监控详情")
-    @PostMapping("/findDetailPage")
-    public Result findDetailPage(@RequestBody QueryPopularFeelingsPageParam param) {
+	@Permission(value = "popular.feelings.delete", tags = "查询舆情监控详情")
+	@PostMapping("/findDetailPage")
+	public Result findDetailPage(@RequestBody QueryPopularFeelingsPageParam param) {
 
-        QueryWrapper<PopularFeelingsPage> wrapper = new QueryWrapper<>();
-        wrapper.select("id", "popular_feelings_id", "url", "title", "spider_time");
-        wrapper.eq("popular_feelings_id", param.getPopularFeelingsId());
+		QueryWrapper<PopularFeelingsPage> wrapper = new QueryWrapper<>();
+		wrapper.select("id", "popular_feelings_id", "url", "title", "keywords", "description", "spider_time", "update_time");
+		wrapper.eq("popular_feelings_id", param.getPopularFeelingsId());
 
-        Page<PopularFeelingsPage> page = param.getPage();
-        page.addOrder(OrderItem.desc("id"));
+		Page<PopularFeelingsPage> page = param.getPage();
+		page.addOrder(OrderItem.desc("id"));
 
-        Page<PopularFeelingsPage> resultData = this.popularFeelingsPageService.page(page, wrapper);
+		Page<PopularFeelingsPage> resultData = this.popularFeelingsPageService.page(page, wrapper);
 
-        return Result.instance(Result.Code.SUCCESS).setData(resultData);
-    }
+		return Result.instance(Result.Code.SUCCESS).setData(resultData);
+	}
 }

src/main/java/com/zhiqiyun/open/router/request/statistics/QueryPopularFeelingsRequest.java

@@ -5,16 +5,16 @@ import com.dliyun.oap.framework.request.AbstractOapRequest;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
-import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
 public class QueryPopularFeelingsRequest extends AbstractOapRequest {
-	@NotBlank
+	@NotNull
 	@ServiceParamField(describe = "页数")
 	private Integer current = 1;
 
-	@NotBlank
+	@NotNull
 	@ServiceParamField(describe = "每页条数")
 	private Integer pageSize = 10;
 

src/main/resources/application.properties

@@ -6,9 +6,9 @@ logger.root.level=DEBUG
 logger.root.path=/tmp/
 ####################### mysql ###############################
 spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver
-spring.datasource.url=jdbc:mysql://rm-bp1e2451m5olnc4g6qo.mysql.rds.aliyuncs.com/liucheng_open?allowMultiQueries=true&useUnicode=true&characterEncoding=utf8&useSSL=false
-spring.datasource.username=liucheng
-spring.datasource.password=liucheng123@
+spring.datasource.url=jdbc:mysql://47.106.38.249:3336/liucheng_open?allowMultiQueries=true&useUnicode=true&characterEncoding=utf8&useSSL=false
+spring.datasource.username=app
+spring.datasource.password=!@qwaszx
 #######################mybatis###############################
 mybatis-plus.config-location=classpath:mybatis/sqlMapConfig.xml
 mybatis-plus.mapper-locations=classpath:mybatis/mappers/*.xml

src/main/resources/db/migration/V1.0.7__popular_feelings.sql

@@ -25,12 +25,15 @@ CREATE TABLE `popular_feelings`
 DROP TABLE IF EXISTS `popular_feelings_page`;
 CREATE TABLE `popular_feelings_page`
 (
-    `id`                  VARCHAR(50) NOT NULL COMMENT 'ID',
+    `id`                  VARCHAR(50)  NOT NULL COMMENT 'ID',
     `popular_feelings_id` BIGINT(20) NOT NULL COMMENT '舆情监控ID',
     `url`                 VARCHAR(500) NOT NULL COMMENT 'URL地址',
-    `title`               TEXT         NOT NULL COMMENT '标题',
+    `title`               VARCHAR(500) NOT NULL COMMENT '标题',
+    `keywords`            VARCHAR(500) NOT NULL COMMENT '关键词',
+    `description`         VARCHAR(500) NOT NULL COMMENT '描述',
     `body_text`           LONGTEXT     NOT NULL COMMENT '网页文本',
     `html`                LONGTEXT     NOT NULL COMMENT '全文本HTML',
     `spider_time`         DATETIME NULL DEFAULT NULL COMMENT '采集时间',
+    `update_time`         DATETIME NULL DEFAULT NULL COMMENT '更新采集时间',
     PRIMARY KEY (`id`)
 ) COMMENT ='舆情采集页面' ENGINE = InnoDB;