Crawler4j Java多线程Web爬虫开源项目

Crawler4j是一个开源的Java类库提供一个用于抓取Web页面的简单接口。可以利用它来构建一个多线程的Web爬虫。

示例代码：

importjava.util.ArrayList;importjava.util.regex.Patter;importedu.uci.ics.crawler4j.crawler.Page;importedu.uci.ics.crawler4j.crawler.WebCrawler;importedu.uci.ics.crawler4j.url.WebURL;publicclassMyCrawlerextedsWebCrawler{       Patterfilters=Patter.compile(".*(\\.(css|js|bmp|gif|jpe?g"               +"|pg|tiff?|mid|mp2|mp3|mp4"               +"|wav|avi|mov|mpeg|ram|m4v|pdf"               +"|rm|smil|wmv|swf|wma|zip|rar|gz))$");              publicMyCrawler(){       }       publicbooleashouldVisit(WebURLurl){               Strighref=url.getURL().toLowerCase();               if(filters.matcher(href).matches()){                       returfalse;               }               if(href.startsWith("https://www.ics.uci.edu/")){                       returtrue;               }               returfalse;       }       publicvoidvisit(Pagepage){               itdocid=page.getWebURL().getDocid();               Strigurl=page.getWebURL().getURL();                       Strigtext=page.getText();               ArrayList<WebURL>liks=page.getURLs();                     }}