webmagic爬取分页列表数据

webmagic是java爬虫框架中比较简单易上手的一个。官网链接：http://webmagic.io/

下面的例子是使用这个框架来爬取工商银行的私人理财推荐分页列表数据。页面链接为：https://mybank.icbc.com.cn/se...$17$TJ&Area_code=0200&requestChannel=302

1.引入webmagic：maven配置

<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.3</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.3</version>
</dependency>

如果项目已经引入slf4j记录日志，则需要在webmagic-extension中排除slf4j-log4j12。如下：

<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.3</version>
    <exclusions>
     <exclusion>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
     </exclusion>
     </exclusions>
</dependency>

2.日志文件配置如下：

log4j.rootLogger=info, ServerDailyRollingFile, stdout
log4j.appender.ServerDailyRollingFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.ServerDailyRollingFile.DatePattern='.'yyyy-MM-dd
log4j.appender.ServerDailyRollingFile.File=/home/myfile/log/mps.log //日志文件路径配置
log4j.appender.ServerDailyRollingFile.layout=org.apache.log4j.PatternLayout
log4j.appender.ServerDailyRollingFile.layout.ConversionPattern=%d - %m%n
log4j.appender.ServerDailyRollingFile.Append=true
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p [%c] %m%n

3.爬虫代码

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;
import java.util.ArrayList;
import java.util.List;
/**
*私行推荐理财
*/
public class PrivateRecommendFinanceSpider implements PageProcessor {
private Logger log = LoggerFactory.getLogger(PrivateRecommendFinanceSpider.class);
// 定义连接失败时，重试机制
private static Site site = Site.me().setRetryTimes(3).setSleepTime(100);
public Site getSite() {
return site;
}
@Override
public void process(Page page) {
log.info("私行推荐理财爬虫开始====");
String pageNum=page.getHtml().xpath("//*[@id=pageturn]/ul/li[3]/span[2]/b/text()").get();
int pageNumInt=Integer.parseInt(pageNum);
for (int i = 1; i < pageNumInt; i++) {
//获取下一页的链接，将当前页数拼接到url上
String nextUrl="https://mybank.icbc.com.cn/servlet/ICBCBaseReqServletNoSession?dse_operationName=per_FinanceCurProListP3NSOp&p3bank_error_backid=120103&pageFlag=0&menuLabel=10$17$TJ&Area_code=0200&requestChannel=302&nowPageNum_turn="+(i+1);
//将下一页链接添加到爬虫队列中
page.addTargetRequest(nextUrl);
}
List<Selectable> nodes = page.getHtml().xpath("//*[@id=datatableModel]/div").nodes(); //获取列表的条数
int length=nodes.size();
System.out.println(length);
List<Object> list = new ArrayList<>();
for (int i = 0; i < length-2; i++) {
PrivateRecommend privateRecommend = new PrivateRecommend();
//根据xpath获取对应节点的内容
privateRecommend.setProductName(page.getHtml().xpath("/html/body/div[1]/div[1]/div[3]/div["+(i+2)+"]/div[2]/div[1]/span[1]/span[1]/a/text()").get());
privateRecommend.setPerformanceBanchmark(page.getHtml().xpath("//*[@id=doublelabel1_"+i+"-content]/text()").get());
privateRecommend.setUpPurchaseAmount(page.getHtml().xpath("//*[@id=doublelabel2_"+i+"-content]/b/text()").get());
privateRecommend.setInvestmentPeriod(page.getHtml().xpath("//*[@id=doublelabel2_"+i+"-content]/b/text()").get());
privateRecommend.setRiskClass(page.getHtml().xpath("//*[@id=tt"+i+"-content]/text()").get());
privateRecommend.setRaisingPeriod(page.getHtml().xpath("/html/body/div[1]/div[1]/div[3]/div["+(i+2)+"]/div[2]/div[1]/span[2]/span/text()").get());
list.add(privateRecommend);
}
//将封装的list对象传到pipline中
page.putField("privateRecommend",list);
}
//执行main方法
public void main(String[] args){
Spider.create(new PrivateRecommendFinanceSpider()).addUrl("https://mybank.icbc.com.cn/servlet/ICBCBaseReqServletNoSession?dse_operationName=per_FinanceCurProListP3NSOp&p3bank_error_backid=120103&pageFlag=0&menuLabel=10$17$TJ&Area_code=0200&requestChannel=302")
.addPipeline(new ConsolePipeline()).run();//pipline中保存数据，此例中consolepipeline直接将内容打印到控制台。可自己定义
log.info("=====私行推荐理财爬虫执行完毕");
}

4.对象的内容

@Data
public class PrivateRecommend {

/**主键*/
private Long id;
/**
 * 名称
 */
private String productName;
/**
 *预期年化收益率
 */
private String performanceBanchmark;
/**
 *起购金额
 */
private String upPurchaseAmount;
/**
 *期限
 */
private String investmentPeriod;
/**
 * 风险等级
 */
private String riskClass;
/**
 * 最近购买开放日
 */
private String raisingPeriod;
/**
 * 更新日期
 */
private String updateTime;

}

5.可能存在的坑
a.可能会存在slf4j日志jar的冲突
b.xpath获取节点上的内容
c.获取列表下一页列表数
d.将下一页链接添加到爬虫队列中