Python+Scrapy爬虫框架之Middleware文件详解

<code># -*- coding: utf-8 -*- # 在这里定义蜘蛛中间件的模型 # Define here the models for your spider middleware # # See documentation in: # https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals # ===========================Spider Middleware============================ # 定义：介于Scrapy引擎和爬虫之间的框架，主要工作是处理蜘蛛的响应输入和请求输出。 # Spider Middleware功能：处理爬虫的请求输入和响应输出 # scrapy已经提供了一些直接使用的中间件，他被SPIDER_MIDDLEWARES_BASE定义： # { # 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, # 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, # 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, # 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, # 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, # } # =================SpiderMiddleware类================== class MaoyanSpiderMiddleware(object): @classmethod # 类方法，参数crawler，可以通过crawler调用settings里的全局参数 def from_crawler(cls, crawler): """ :param crawler: 获取settings里的全局参数，如crawler.settings.get(参数) """ s = cls() # 调用spider_opened函数进行爬取数据并对该函数发送该信号。该信号一般用来分配spider的资源 crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) # 调用spider_closed函数进行关闭爬虫并对该函数发送该信号。该信号用来释放spider在spider_opened时占用的资源。 # crawler.signals.connect(s.spider_closed, signal=signals.spider_closed) return s # 当返回来的response被Spider Middleware处理时，该方法被调用 def process_spider_input(self, response, spider): """ :param response: 被Spider Middleware处理的response对象 :param spider: 返回response对应的spider对象 """ return None # 当spider处理response对象的结果后，该方法被调用 def process_spider_output(self, response, result, spider): """ :param response: 被spider处理后得到结果的response对象 :param result: result包含Item或request对象的可迭代对象，即spider返回的response结果 :param spider: 返回response对象的spider对象 """ # 遍历返回的可迭代对象 for i in result: yield i # 当spider的process_spider_input和process_spider_output发生异常时调用该方法 def process_spider_exception(self, response, exception, spider): """ :param response: 异常被抛出时被处理的response对象 :param exception: 抛出的异常 :param spider: 抛出该异常的spider对象 """ pass # 以spider启动的request为参数调用该方法,返回一个request可迭代对象 def process_start_requests(self, start_requests, spider): """ :param start_requests: 开始请求的可迭代对象 :param spider: 开始请求所对应的spider对象 """ # 遍历可迭代对象 for r in start_requests: yield r # 当spider开启时调用该函数，说明开始爬取数据并分配spider的资源 def spider_opened(self, spider): """ :param spider: 开始爬取的spider对象 """ spider.logger.info('Spider opened: %s' % spider.name) # # 当某个spider被关闭时，说明关闭该爬虫并释放spider在spider_opened时占用的资源。 # def spider_closed(self, spider): # """ # :param spider: 开始爬取的spider对象 # """ # spider.logger.info('Spider opened:%s'%spider.name) # ======================Downloader Middleware======================== # 定义：位于Scrapy引擎和下载器之间的框架，主要是处理Scrapy引擎与下载器之间的请求及响应。见scrapy框架图 # Downloader Middleware功能：可以修改User-Agent、处理重定向、设置代理、失败重试、设置Cookies等 # scrapy已经提供了一些直接使用的中间件，他被DOWNLOADER_MIDDLEWARES_BASE定义： # { # 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, # 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, # 'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350, # 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, # 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, # 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, # 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, # 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, # 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, # 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, # 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, # 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, # 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, # 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, # } # ===============DownloaderMiddleware类================= class MaoyanDownloaderMiddleware(object): @classmethod # 类方法，参数crawler，可以通过crawler调用settings里的全局参数 def from_crawler(cls, crawler): """ :param crawler: 获取settings里的全局参数，如crawler.settings.get(参数) """ s = cls() # 调用spider_opened函数进行爬取数据并对该函数发送该信号。该信号一般用来分配spider的资源 crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) # 调用spider_closed函数进行关闭爬虫并对该函数发送该信号。该信号用来释放spider在spider_opened时占用的资源。 # crawler.signals.connect(s.spider_closed, signal=signals.spider_closed) return s # request被scrapy从调度器调度给Downloader Middleware之前调用该方法对request对象进行处理 def process_request(self, request, spider): """ :param request: 就是scrapy从调度器调度出来的request对象 :param spider: 就是scrapy调度出来的request对象的spider对象 """ return None # request对象被Downloader Middleware执行后返回response是才调用该方法对response对象进行处理 def process_response(self, request, response, spider): """ :param request: 调度出来被Downloader Middleware处理的request对象 :param response: Downloader Middleware处理request对象返回后的response对象 :param spider: response返回来的spider对象 """ return response # 当process_request和process_response发生异常时调用 def process_exception(self, request, exception, spider): """ :param request: 产生异常的request对象 :param exception: 抛出的异常对象 :param spider: 产生异常的request对象的spider对象 """ pass # 当spider开启时调用该函数，说明开始爬取数据并分配spider的资源 def spider_opened(self, spider): """ :param spider: 开始爬取的spider对象 """ spider.logger.info('Spider opened: %s' % spider.name) # # 当某个spider被关闭时，说明关闭该爬虫并释放spider在spider_opened时占用的资源。 # def spider_closed(self, spider): # """ # :param spider: 开始爬取的spider对象 # """ # spider.logger.info('Spider opened: %s' % spider.name)/<code>