WebCollector-Pytho
WebCollector-Pytho是一个无须配置、便于二次开发的Pytho爬虫框架（内核），它提供精简的的API，只需少量代码即可实现一个功能强大的爬虫。
WebCollectorJava版本WebCollectorJava版相比WebCollector-Pytho具有更高的效率: https://github.com/CrawlScript/WebCollector
安装pip安装命令pip istall https://github.com/CrawlScript/WebCollector-Pytho/archive/master.zip示例Basicdemo_auto_ews_crawler.py
demo_maual_ews_crawler.py
快速入门自动探测URLdemo_auto_ews_crawler.py:
# codig=utf-8import webcollector as wcclass NewsCrawler(wc.RamCrawler): def __iit__(self): super().__iit__(auto_detect=True) self.um_threads = 10 self.add_seed("https://github.blog/") self.add_regex("+https://github.blog/[0-9]+.*") self.add_regex("-.*#.*") # do ot detect urls that cotai "#" def visit(self, page, detected): if page.match_url("https://github.blog/[0-9]+.*"): title = page.select("h1.lh-codesed")[0].text.strip() cotet = page.select("div.markdow-body")[0].text.replace("\", " ").strip() prit("\URL: ", page.url) prit("TITLE: ", title) prit("CONTENT: ", cotet[:50], "...")crawler = NewsCrawler()crawler.start(10)手动探测URLdemo_maual_ews_crawler.py:
# codig=utf-8import webcollector as wcclass NewsCrawler(wc.RamCrawler): def __iit__(self): super().__iit__(auto_detect=False) self.um_threads = 10 self.add_seed("https://github.blog/") def visit(self, page, detected): detected.exted(page.liks("https://github.blog/[0-9]+.*")) if page.match_url("https://github.blog/[0-9]+.*"): title = page.select("h1.lh-codesed")[0].text.strip() cotet = page.select("div.markdow-body")[0].text.replace("\", " ").strip() prit("\URL: ", page.url) prit("TITLE: ", title) prit("CONTENT: ", cotet[:50], "...")crawler = NewsCrawler()crawler.start(10)用detected_filter插件过滤探测到的URLdemo_detected_filter.py:
# codig=utf-8import webcollector as wcfrom webcollector.filter import Filterimport reclass RegexDetectedFilter(Filter): def filter(self, crawl_datum): if re.fullmatch("https://github.blog/2019-02.*", crawl_datum.url): retur crawl_datum else: prit("filtered by detected_filter: {}".format(crawl_datum.brief_ifo())) retur Noeclass NewsCrawler(wc.RamCrawler): def __iit__(self): super().__iit__(auto_detect=True, detected_filter=RegexDetectedFilter()) self.um_threads = 10 self.add_seed("https://github.blog/") def visit(self, page, detected): detected.exted(page.liks("https://github.blog/[0-9]+.*")) if page.match_url("https://github.blog/[0-9]+.*"): title = page.select("h1.lh-codesed")[0].text.strip() cotet = page.select("div.markdow-body")[0].text.replace("\", " ").strip() prit("\URL: ", page.url) prit("TITLE: ", title) prit("CONTENT: ", cotet[:50], "...")crawler = NewsCrawler()crawler.start(10)用RedisCrawler进行可断点的采集（可在关闭后恢复）demo_redis_crawler.py:
# codig=utf-8from redis import StrictRedisimport webcollector as wcclass NewsCrawler(wc.RedisCrawler): def __iit__(self): super().__iit__(redis_cliet=StrictRedis("127.0.0.1"), db_prefix="ews", auto_detect=True) self.um_threads = 10 self.resumable = True # you ca resume crawlig after shutdow self.add_seed("https://github.blog/") self.add_regex("+https://github.blog/[0-9]+.*") self.add_regex("-.*#.*") # do ot detect urls that cotai "#" def visit(self, page, detected): if page.match_url("https://github.blog/[0-9]+.*"): title = page.select("h1.lh-codesed")[0].text.strip() cotet = page.select("div.markdow-body")[0].text.replace("\", " ").strip() prit("\URL: ", page.url) prit("TITLE: ", title) prit("CONTENT: ", cotet[:50], "...")crawler = NewsCrawler()crawler.start(10)用Requests定制Http请求demo_custom_http_request.py:
#codig=utf-8importwebcollectoraswcfromwebcollector.modelimportPagefromwebcollector.plugi.etimportHttpRequesterimportrequestsclassMyRequester(HttpRequester):defget_respose(self,crawl_datum):#customhttprequestheaders={"User-Aget":"Mozilla/5.0(WidowsNT10.0;Wi64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/76.0.3809.100Safari/537.36"}prit("sedigrequestwithMyRequester")#sedrequestadgetresposerespose=requests.get(crawl_datum.url,headers=headers)#updatecodecrawl_datum.code=respose.status_code#wraphttpresposeasaPageobjectpage=Page(crawl_datum,respose.cotet,cotet_type=respose.headers["Cotet-Type"],http_charset=respose.ecodig)returpageclassNewsCrawler(wc.RamCrawler):def__iit__(self):super().__iit__(auto_detect=True)self.um_threads=10#setrequestertoeableMyRequesterself.requester=MyRequester()self.add_seed("https://github.blog/")self.add_regex("+https://github.blog/[0-9]+.*")self.add_regex("-.*#.*")#dootdetecturlsthatcotai"#"defvisit(self,page,detected):ifpage.match_url("https://github.blog/[0-9]+.*"):title=page.select("h1.lh-codesed")[0].text.strip()cotet=page.select("div.markdow-body")[0].text.replace("\","").strip()prit("\URL:",page.url)prit("TITLE:",title)prit("CONTENT:",cotet[:50],"...")crawler=NewsCrawler()crawler.start(10)
评论