python爬虫29 | 使用scrapy爬取糗事百科的例子，告诉你它有多厉害！

conda install -c conda-forge scrapy

pip install Scrapy

lxml
parsel
w3lib
twisted
cryptography and pyOpenSSL

 scrapy startproject qiushibaike

class QiushiSpider(scrapy.Spider):

  name = "qiushibaike"

    def start_requests(self):
        urls = [
            'https://www.qiushibaike.com/text/page/1/',
            'https://www.qiushibaike.com/text/page/2/',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = 'qiushi-%s.html' % page
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log('Saved file %s' % filename)

 cd qiushibaike/
scrapy crawl qiushibaike

 haders = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/73.0.3683.86 Chrome/73.0.3683.86 Safari/537.36'
    }

    def start_requests(self):
        urls = [
            'https://www.qiushibaike.com/text/page/12/',
            'https://www.qiushibaike.com/text/page/2/',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse, headers=self.haders)

scrapy shell /home/wistbean/PycharmProjects/spider/qiushibaike/qiushi-1.html

>>> content_left_div = response.xpath('//*[@id="content-left"]')

>>> content_list_div = content_left_div.xpath('./div')

content_div = content_list_div[0]

>>> author = content_div.xpath('./div/a[2]/h2/text()').get()
>>> print(author)

街角*

>>> content = content_div.xpath('./a/div/span/text()').getall()
>>> print(content)
['nnn刚上小学一年级时，一次肚子有点不舒服拉肚子，跑到厕所后运动裤上面的绳子竟然打了死结，在厕所里面如无头苍蝇一样乱转。', '这时看到老师在旁边上蹲坑，也不顾难为情了'此时老师蹲在蹲坑躲闪不及，浑身上下星星点点，屎喷了满脸如同长满麻子般！', '又是开心的一天，没有拉裤子！nn']
>>> 

 def parse(self, response):

        content_left_div = response.xpath('//*[@id="content-left"]')
        content_list_div = content_left_div.xpath('./div')

        for content_div in content_list_div:
            yield {
                'author': content_div.xpath('./div/a[2]/h2/text()').get(),
                'content': content_div.xpath('./a/div/span/text()').getall(),
            }

scrapy crawl qiushibaike

scrapy crawl qiushibaike -o qiushibaike.json

FEED_EXPORT_ENCODING = 'utf-8'