python爬虫

class ScrapyJsonPipeline():
    def __init__(self):
        self.file =codecs.open('scrapy_jobbole.json','w',encoding="utf-8")

    def process_item(self, item, spider):
        lines = json.dumps(dict(item),ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def close_spider(self,spider):
        self.file.close()

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/9/6 17:58
# @Author: jecht
# @File : get_md5.py
import hashlib

def get_md5(url):
    if isinstance(url,str):
        url = url.encode("utf-8")
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()

if __name__ == "__main__":
    print (get_md5("http://jobbole.com"))

page_urls = filter(lambda x:True if x.startswith("https") else False, page_urls)

raink_item = ",".join(str(i.stip()) for i in self["raink_item"])

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/8/29 2:33
# @Author: jecht
# @File : spider_main.py

from baidubaike_spider import url_manager, url_download, url_parser, spider_output

class SpiderMain(object):
    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.downloader = url_download.UrlDownload()
        self.parser = url_parser.UrlParser()
        self.output = spider_output.SpiderOutput()
    def craw(self, root_url):
        count = 1
        self.urls.add_new_url(root_url)
        #当有待爬取的url时
        while self.urls.has_new_url():
            try:
                #获取新的url
                new_url = self.urls.get_new_url()
                print('craw %d : %s' %(count, new_url))
                #下载新的url页面
                html_count = self.downloader.download(new_url)
                #将url和所在的页面内容通过parser函数得到新的new_url数据和new_data数据
                new_urls,new_data = self.parser.parser(new_url,html_count)
                #将新的url数据补充到url管理器当中
                self.urls.add_new_urls(new_urls)
                #将新的数据存放到output文件当中
                self.output.collect_data(new_data)
                if count == 1000:
                    break
                count = count + 1
            except:
                print('craw failed!')
        self.output.output_html()

if __name__=="__main__":
    root_url = "http://baike.baidu.com/view/21087.htm"
    obj_spider = SpiderMain()
    obj_spider.craw(root_url)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/8/29 2:35
# @Author: jecht
# @File : url_manager.py

class UrlManager(object):
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()
    #管理器中新添加的url
    def add_new_url(self, url):
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)
    #管理器中新添加批量的url
    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)
    #判断管理器中是否有新的待爬取url
    def has_new_url(self):
        return len(self.new_urls) != 0
    #从管理器中获取一个新的待爬取的url
    def get_new_url(self):
        new_url =self.new_urls.pop()
        self.old_urls.add(new_url)
        return new_url

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/8/29 2:36
# @Author: jecht
# @File : url_download.py

import urllib.request

class UrlDownload(object):
    def download(self, url):
        if url is None:
            return None
        response =  urllib.request.urlopen(url)
        if response.getcode() != 200:
            return None
        return response.read()

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/8/29 15:41
# @Author: jecht
# @File : url_parser.py

import re
from anaconda_project.plugins.network_util import urlparse
from bs4 import BeautifulSoup

class UrlParser(object):

    def _get_new_urls(self,page_url,soup):
        new_urls =set()
        links = soup.find_all('a',href=re.compile(r"/view/\d+\.htm"))
        for link in links:
            new_url = link['href']
            #拼接url
            new_full_url = urlparse.urljoin(page_url,new_url)
            new_urls.add(new_full_url)
        return new_urls

    def _get_new_data(self, page_url, soup):
        res_data={}
        #url
        res_data['url'] = page_url

        #<div class="lemma-summary" label-module="lemmaSummary">
        title_node =soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find("h1")
        res_data['title'] = title_node.get_text()

        summary_node = soup.find('div', class_="lemma-summary")
        res_data['summary'] = summary_node.get_text()

        return res_data

    def parser(self, page_url, html_count):
        if page_url is None or html_count is None:
            return
        soup = BeautifulSoup(html_count,'html.parser',from_encoding='utf-8')
        new_urls = self._get_new_urls(page_url,soup)
        new_data = self._get_new_data(page_url,soup)
        return new_urls,new_data

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/8/29 12:33
# @Author: jecht
# @File : spider_output.py

class SpiderOutput(object):

    def __init__(self):
        self.datas = []

    def collect_data(self, data):
        if data is None:
            return
        self.datas.append(data)

    def output_html(self):
        fout = open('output.html','w')

        fout.write("<html>")
        fout.write("<body>")
        fout.write("<table>")
        #ascii
        for data in self.datas:
            fout.write("<tr>")
            fout.write("<td>%s</td>" % data['url'])
            fout.write("<td>%s</td>" % data['title'].encode('utf-8'))
            fout.write("<td>%s</td>" % data['summary'].encode('utf-8'))
            fout.write("</tr>")
        fout.write("</table>")
        fout.write("</body>")
        fout.write("</html>")

        fout.close()

class CrawlSpider(Spider):

    rules = ()

    def __init__(self, *a, **kw):
        super(CrawlSpider, self).__init__(*a, **kw)
        self._compile_rules()

    def parse(self, response):
        return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)

    def parse_start_url(self, response):
        return []

    def process_results(self, response, results):
        return results

    def _build_request(self, rule, link):
        r = Request(url=link.url, callback=self._response_downloaded)
        r.meta.update(rule=rule, link_text=link.text)
        return r

    def _requests_to_follow(self, response):
        if not isinstance(response, HtmlResponse):
            return
        seen = set()
        for n, rule in enumerate(self._rules):
            links = [lnk for lnk in rule.link_extractor.extract_links(response)
                     if lnk not in seen]
            if links and rule.process_links:
                links = rule.process_links(links)
            for link in links:
                seen.add(link)
                r = self._build_request(n, link)
                yield rule.process_request(r)

    def _response_downloaded(self, response):
        rule = self._rules[response.meta['rule']]
        return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)

    def _parse_response(self, response, callback, cb_kwargs, follow=True):
        if callback:
            cb_res = callback(response, **cb_kwargs) or ()
            cb_res = self.process_results(response, cb_res)
            for requests_or_item in iterate_spider_output(cb_res):
                yield requests_or_item

        if follow and self._follow_links:
            for request_or_item in self._requests_to_follow(response):
                yield request_or_item

    def _compile_rules(self):
        def get_method(method):
            if callable(method):
                return method
            elif isinstance(method, six.string_types):
                return getattr(self, method, None)

        self._rules = [copy.copy(r) for r in self.rules]
        for rule in self._rules:
            rule.callback = get_method(rule.callback)
            rule.process_links = get_method(rule.process_links)
            rule.process_request = get_method(rule.process_request)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
        spider._follow_links = crawler.settings.getbool(
            'CRAWLSPIDER_FOLLOW_LINKS', True)
        return spider

    def set_crawler(self, crawler):
        super(CrawlSpider, self).set_crawler(crawler)
        self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

def start_requests(self):
    cls = self.__class__
    if method_is_overridden(cls, Spider, 'make_requests_from_url'):
        warnings.warn(
            "Spider.make_requests_from_url method is deprecated; it "
            "won't be called in future Scrapy releases. Please "
            "override Spider.start_requests method instead (see %s.%s)." % (
                cls.__module__, cls.__name__
            ),
        )
        for url in self.start_urls:
            yield self.make_requests_from_url(url)
    else:
        for url in self.start_urls:
            yield Request(url, dont_filter=True)

def _parse_response(self, response, callback, cb_kwargs, follow=True):
    if callback:
        cb_res = callback(response, **cb_kwargs) or ()
        cb_res = self.process_results(response, cb_res)
        for requests_or_item in iterate_spider_output(cb_res):
            yield requests_or_item
    if follow and self._follow_links:
        for request_or_item in self._requests_to_follow(response):
            yield request_or_item

def parse_start_url(self, response):
    return []

def process_results(self, response, results):
    return results

rules = (
    Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
)

def _requests_to_follow(self, response):
    if not isinstance(response, HtmlResponse):
        return
    seen = set()
    for n, rule in enumerate(self._rules):
        links = [lnk for lnk in rule.link_extractor.extract_links(response)
                 if lnk not in seen]
        if links and rule.process_links:
            links = rule.process_links(links)
        for link in links:
            seen.add(link)
            r = self._build_request(n, link)
            yield rule.process_request(r)

def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
             tags=('a', 'area'), attrs=('href',), canonicalize=False,
             unique=True, process_value=None, deny_extensions=None, restrict_css=(),
             strip=True)

def _compile_rules(self):
    def get_method(method):
        if callable(method):
            return method
        elif isinstance(method, six.string_types):
            return getattr(self, method, None)

    self._rules = [copy.copy(r) for r in self.rules]
    for rule in self._rules:
        rule.callback = get_method(rule.callback)
        rule.process_links = get_method(rule.process_links)
        rule.process_request = get_method(rule.process_request)

def _build_request(self, rule, link):
    r = Request(url=link.url, callback=self._response_downloaded)
    r.meta.update(rule=rule, link_text=link.text)
    return r

def _response_downloaded(self, response):
    rule = self._rules[response.meta['rule']]
    return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)

rules = (
    Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
)

class Rule(object):

def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
    self.link_extractor = link_extractor
    self.callback = callback
    self.cb_kwargs = cb_kwargs or {}
    self.process_links = process_links
    self.process_request = process_request
    if follow is None:
        self.follow = False if callback else True
    else:
        self.follow = follow

class LxmlLinkExtractor(FilteringLinkExtractor):

    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                 tags=('a', 'area'), attrs=('href',), canonicalize=False,
                 unique=True, process_value=None, deny_extensions=None, restrict_css=(),
                 strip=True):

def extract_links(self, response):
    base_url = get_base_url(response)
    if self.restrict_xpaths:
        docs = [subdoc
                for x in self.restrict_xpaths
                for subdoc in response.xpath(x)]
    else:
        docs = [response.selector]
    all_links = []
    for doc in docs:
        links = self._extract_links(doc, response.url, response.encoding, base_url)
        all_links.extend(self._process_links(links))
    return unique_list(all_links)