python爬虫30 | scrapy后续，把「糗事百科」的段子爬下来然后存到数据库中

next_page = response.xpath('//*[@id="content-left"]/ul/li[last()]/a').attrib['href']

  if next_page is not None:
      yield response.follow(next_page, callback=self.parse)

# if next_page is not None:
      # next_page = response.urljoin(next_page)
      # yield scrapy.Request(next_page, callback=self.parse)

import scrapy

class QiushibaikeItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    author = scrapy.Field()
    content = scrapy.Field()
    _id = scrapy.Field()

    def parse(self, response):

        content_left_div = response.xpath('//*[@id="content-left"]')
        content_list_div = content_left_div.xpath('./div')

        for content_div in content_list_div:
            item = QiushibaikeItem()
            item['author'] = content_div.xpath('./div/a[2]/h2/text()').get()
            item['content'] = content_div.xpath('./a/div/span/text()').getall()
            item['_id'] = content_div.attrib['id']
            yield item
        next_page = response.xpath('//*[@id="content-left"]/ul/li[last()]/a').attrib['href']

        if next_page is not None:
            yield response.follow(next_page, callback=self.parse)

class QiushibaikePipeline(object):

    def __init__(self):
        self.connection = pymongo.MongoClient('localhost', 27017)
        self.db = self.connection.scrapy  
        self.collection = self.db.qiushibaike 

    def process_item(self, item, spider):
        if not self.connection or not item:
            return
        self.collection.save(item)

    def __del__(self):
        if self.connection:
            self.connection.close()

# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'qiushibaike.pipelines.QiushibaikePipeline': 300,
}

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/73.0.3683.86 Chrome/73.0.3683.86 Safari/537.36'

scrapy crawl qiushibaike