scrapy使用seleium和中间件

import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from wangyiPro.items import WangyiproItem
"""
爬取网易国内和国际新闻标题和内容
"""
class WangyiSpider(scrapy.Spider):
    name = ‘wangyi‘
    # allowed_domains = [‘www.163.com‘]
    start_urls = [‘https://news.163.com/domestic/‘,‘https://news.163.com/world/‘]


    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_argument(‘--window-position=0,0‘);  # chrome 启动初始位置
        options.add_argument(‘--window-size=1080,800‘);  # chrome 启动初始大小
        self.browser = webdriver.Chrome(executable_path=‘C://xx//chromedriver.exe‘ ,chrome_options=options)

    def parse(self, response):
       div_list =  response.xpath(‘//div[@class="ndi_main"]/div‘)
       for div_item in div_list:
           title = div_item.xpath(‘./div/div[1]/h3/a/text()‘).extract_first()
           new_detail_url=div_item.xpath(‘./div/div[1]/h3/a/@href‘).extract_first()
           item = WangyiproItem()
           item[‘title‘] = title

           # 对于新闻详情页发起request
           yield scrapy.Request(url= new_detail_url,callback=self.parse_detail,meta={‘item‘:item}) # 请求传参item

    # 解析新闻内容
    def parse_detail(self,response):
        content = response.xpath(‘//*[@id="endText"]//text()‘).extract()
        content = ‘‘.join(content)
        item = response.meta[‘item‘]
        item[‘content‘] = content.strip()

        yield item

    def closed(self,spider):
        self.browser.quit()

from scrapy import signals
from time import sleep
from scrapy.http import HtmlResponse
class WangyiproDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    # 拦截响应对象进行篡改
    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.
        #挑选指定的响应对象进行篡改
        #通过url指定request
        #通过request指定response
        #spider爬虫对象
        bro = spider.browser # 获取爬虫类定义的浏览器对象
        if request.url in spider.start_urls:
            #response   # 进行篡改 实例化新的响应对象（包含动态加载的新闻数据）替代原来的旧响应对象
            # 基于seleium便捷获取动态数据
            bro.get(request.url)
            sleep(3)
            bro.execute_script(‘window.scrollTo(0, document.body.scrollHeight)‘)
            sleep(1)
            page_text = bro.page_source # 包含了动态加载对象
            new_response = HtmlResponse(url=request.url,body=page_text,encoding="utf-8",request=request)

            return new_response
        else:
            # response # 其他请求
        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
            return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info(‘Spider opened: %s‘ % spider.name)

import pymysql

class WangyiproPipeline(object):
    # 构造方法
    def __init__(self):
        self.conn = None  # 定义一个文件描述符属性
        self.cursor = None
        self.num = 0

    # 下列都是在重写父类的方法：
    # 开始爬虫时，执行一次
    def open_spider(self, spider):
        self.conn = pymysql.Connect(host=‘192.168.xx.xx‘, port=3306, user=‘root‘, password=‘xx‘, db=‘xx_db‘,
                                    charset=‘utf8‘)
        print(‘爬虫数据库开始‘)

    # 专门处理item对象
    # 因为该方法会被执行调用多次，所以文件的开启和关闭操作写在了另外两个只会各自执行一次的方法中。
    def process_item(self, item, spider):
        author = item[‘title‘]
        content = item[‘content‘]
        self.cursor = self.conn.cursor()
        try:

            self.cursor.execute(‘insert into qiubai values(%s,%s)‘, (author, content))
            self.conn.commit()
        except Exception as e:
            print(e,content[0,20])
            self.conn.rollback()
        return item

    def close_spider(self, spider):
        print(‘爬虫数据库结束‘)
        self.cursor.close()
        self.conn.close()

class WangyiproItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()
    pass

USER_AGENT = ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36‘  # 伪装请求载体身份
# Obey robots.txt rules
# ROBOTSTXT_OBEY = True
ROBOTSTXT_OBEY = False  #可以忽略或者不遵守robots协议
#只显示指定类型的日志信息
LOG_LEVEL=‘ERROR‘

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘,
#   ‘Accept-Language‘: ‘en‘,
#}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    ‘wangyiPro.middlewares.WangyiproSpiderMiddleware‘: 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   ‘wangyiPro.middlewares.WangyiproDownloaderMiddleware‘: 543,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    ‘scrapy.extensions.telnet.TelnetConsole‘: None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   ‘wangyiPro.pipelines.WangyiproPipeline‘: 300,
}