{"id":5059,"date":"2019-05-13T08:39:48","date_gmt":"2020-04-22T06:37:59","guid":{"rendered":"https:\/\/vip.fxxkpython.com\/?p=5059"},"modified":"2021-08-31T03:27:19","modified_gmt":"2021-08-30T19:27:19","slug":"360875e2f9","status":"publish","type":"post","link":"https:\/\/vip.fxxkpython.com\/?p=5059","title":{"rendered":"python\u722c\u866b30 | scrapy\u540e\u7eed\uff0c\u628a\u300c\u7cd7\u4e8b\u767e\u79d1\u300d\u7684\u6bb5\u5b50\u722c\u4e0b\u6765\u7136\u540e\u5b58\u5230\u6570\u636e\u5e93\u4e2d"},"content":{"rendered":"
\n

\n

\u4e0a\u56de\u6211\u4eec\u8bf4\u5230<\/p>\n

\n

python\u722c\u866b29 | \u4f7f\u7528scrapy\u722c\u53d6\u7cd7\u4e8b\u767e\u79d1\u7684\u4f8b\u5b50\uff0c\u544a\u8bc9\u4f60\u5b83\u6709\u591a\u5389\u5bb3\uff01<\/a><\/p>\n

\n

WOW\uff01\uff01<\/p>\n

\n

scrapy<\/a><\/p>\n

\n

awesome\uff01\uff01<\/p>\n

\n

\u600e\u4e48\u4f1a\u6709\u8fd9\u4e48\u725b\u903c\u7684\u6846\u67b6<\/p>\n

\n

wow\uff01\uff01<\/p>\n

awesome\uff01\uff01<\/span><\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

\u7528 scrapy \u6765\u722c\u53d6\u6570\u636e<\/p>\n

\n

\u5c82\uff01\u4e0d\uff01\u662f\uff01\u975e\uff01\u5e38\uff01\u723d\uff01<\/p>\n

\n

wow\uff01\uff01<\/p>\n

\n

\u63a5\u4e0b\u6765\u5c31\u662f\u6211\u72ec\u4eab\u7684moment<\/p>\n

\n

\u54e6\u4e0d<\/p>\n

\n

\u63a5\u4e0b\u6765\u5c31\u662f<\/p>\n

\n

\u5b66\u4e60 python \u7684\u6b63\u786e\u59ff\u52bf<\/span><\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

\n

\u6211\u4eec\u5df2\u7ecf\u521b\u5efa\u4e86\u722c\u53d6\u7cd7\u4e8b\u767e\u79d1\u7684\u9879\u76ee<\/p>\n

\n

\u5e76\u4e14\u628a\u7cd7\u4e8b\u767e\u79d1\u7684\u524d\u4e24\u9875\u7684\u4f5c\u8005\u548c\u6bb5\u5b50\u722c\u53d6\u5230 json \u6587\u4ef6\u4e86<\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

\u8fd9\u6b21<\/p>\n

\n

\u6211\u4eec\u5c06\u6211\u4eec\u8981\u722c\u53d6\u6240\u6709\u7684\u6570\u636e<\/p>\n

\n

\u4f7f\u7528 scrapy \u5b58\u50a8\u5230 mangodb \u4e2d<\/p>\n

\n

\u5728\u6b64\u4e4b\u524d\u8fd8\u662f\u5148\u4ecb\u7ecd\u4e00\u4e0b\u6211\u4eec\u4f7f\u7528 scrapy \u521b\u5efa\u51fa\u6765\u7684\u6587\u4ef6\u76ee\u5f55<\/p>\n

\n

\u5404\u4e2a\u6587\u4ef6\u4ee3\u8868\u7684\u90fd\u662f\u5565\u610f\u601d<\/p>\n

\n

\u514d\u5f97\u53c8\u6709\u4e9b b \u53cb\u5f53\u573a\u61f5\u903c<\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

\u6211\u4eec\u4ece\u4e0a\u5f80\u4e0b\u4f9d\u4e2a\u4ecb\u7ecd\u4e00\u4e0b<\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

\u8fd9\u4e2a spiders \u76ee\u5f55<\/span>\u5462<\/p>\n

\n

\u5c31\u662f\u7528\u6765\u5b58\u653e\u6211\u4eec\u5199\u722c\u866b\u6587\u4ef6\u7684\u5730\u65b9<\/p>\n

\n

items.py<\/span><\/p>\n

\n

\u5c31\u662f\u7528\u6765\u5b9a\u4e49\u6211\u4eec\u8981\u5b58\u50a8\u6570\u636e\u7684\u5b57\u6bb5<\/p>\n

\n

middlewares.py\u00a0<\/span><\/p>\n

\n

\u5c31\u662f\u4e2d\u95f4\u4ef6\uff0c\u5728\u8fd9\u91cc\u9762\u53ef\u4ee5\u505a\u4e00\u4e9b\u5728\u722c\u866b\u8fc7\u7a0b\u4e2d\u60f3\u5e72\u7684\u4e8b\u60c5\uff0c\u6bd4\u5982\u722c\u866b\u5728\u54cd\u5e94\u7684\u65f6\u5019\u4f60\u53ef\u4ee5\u505a\u4e00\u4e9b\u64cd\u4f5c<\/p>\n

\n

pipelines.py<\/span><\/p>\n

\n

\u8fd9\u662f\u6211\u4eec\u7528\u6765\u5b9a\u4e49\u4e00\u4e9b\u5b58\u50a8\u4fe1\u606f\u7684\u6587\u4ef6\uff0c\u6bd4\u5982\u6211\u4eec\u8981\u8fde\u63a5 MySQL\u6216\u8005 MongoDB \u5c31\u53ef\u4ee5\u5728\u8fd9\u91cc\u5b9a\u4e49<\/p>\n

\n

settings.py<\/span><\/p>\n

\u00a0<\/span><\/p>\n

\u8fd9\u4e2a\u6587\u4ef6\u7528\u6765\u5b9a\u4e49\u6211\u4eec\u7684\u5404\u79cd\u914d\u7f6e\uff0c\u6bd4\u5982\u914d\u7f6e\u8bf7\u6c42\u5934\u4fe1\u606f\u7b49<\/p>\n

\n

\u4ee5\u4e0a\u5c31\u662f scrapy \u751f\u6210\u7684\u76ee\u5f55\u4e2d\u4e3b\u8981\u6587\u4ef6\u7684\u4f5c\u7528<\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

ok<\/p>\n

\n

\u63a5\u4e0b\u6765\u6211\u4eec\u5c31\u8fdb\u5165\u4ee3\u7801\u4e2d<\/p>\n

\n

\u6211\u4eec\u4e0a\u6b21\u521b\u5efa\u4e86\u00a0QiushiSpider \u6765\u5199\u6211\u4eec\u7684\u722c\u866b<\/span><\/p>\n

\u00a0<\/span><\/p>\n

\u5f53\u65f6\u6211\u4eec\u53ea\u662f\u83b7\u53d6\u4e86\u524d\u4e24\u9875\u7684\u6570\u636e<\/p>\n

\n

\u6211\u4eec\u8981\u83b7\u53d6\u6240\u6709\u9875\u9762\u7684\u6570\u636e\u600e\u4e48\u73a9\u5462<\/p>\n

\n

\u6253\u5f00\u7cd7\u4e8b\u767e\u79d1\u7684\u94fe\u63a5\u53ef\u4ee5\u770b\u5230<\/p>\n

\n

\"python\u722c\u866b30<\/p>\n

\n

13 \u9875\u7684\u6570\u636e<\/p>\n

\n

\u5176\u5b9e\u6309\u7167\u4ee5\u524d\u6211\u4eec\u76f4\u63a5\u5199\u4e2a for \u5faa\u73af\u5c31\u53ef\u4ee5\u4e86<\/p>\n

\n

\u4e0d\u8fc7\u6211\u4eec\u8fd9\u6b21\u8fd8\u53ef\u4ee5\u4f7f\u7528 scrapy \u7684\u00a0follow \u51fd\u6570<\/span><\/p>\n

\u00a0<\/span><\/p>\n

\u5177\u4f53\u4f7f\u7528\u662f\u8fd9\u6837\u7684<\/span><\/p>\n

\u00a0<\/span><\/p>\n

\u6211\u4eec\u5148\u83b7\u53d6\u4e0b\u4e00\u9875\u7684\u94fe\u63a5<\/span><\/p>\n

\u00a0<\/span><\/p>\n

\"python\u722c\u866b30<\/p>\n

\n

\u7531\u4e8e\u4e0b\u4e00\u9875\u8fd9\u4e2a\u6309\u94ae\u90fd\u662f\u5728\u6700\u540e\u4e00\u4e2a li \u6807\u7b7e\u4e2d\u7684<\/p>\n

\n

\u6240\u4ee5\u7528 xpath \u83b7\u53d6\u5c31\u8fd9\u6837<\/p>\n

\n

\n
next_page<\/span> = response.xpath('\/\/*[@id=\"content-left\"]\/ul\/li[last()]\/a'<\/span>).attrib['href'<\/span>]<\/span><\/code><\/pre>\n<\/section>\n
\n
\u63a5\u7740\u6211\u4eec\u5c31\u53ef\u4ee5\u8ba9\u5b83\u53bb\u8bf7\u6c42\u4e0b\u4e00\u9875\u7684\u5185\u5bb9\u6570\u636e\u4e86<\/p>\n
\n
\n
\u00a0\u00a0if<\/span>\u00a0next_page\u00a0is<\/span>\u00a0not<\/span>\u00a0None<\/span>:\n<\/span><\/code>\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0yield<\/span>\u00a0response.follow(next_page,\u00a0callback=self.parse)<\/span><\/code><\/pre>\n<\/section>\n
\n
\u4f60\u4e5f\u53ef\u4ee5\u7528 urljoin \u7684\u65b9\u5f0f<\/p>\n
\n
\n
# if<\/span> next_page is not None:\n<\/span><\/span><\/code>\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0# next_page\u00a0=\u00a0response.urljoin(next_page)\n<\/span><\/span><\/code>\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0# yield\u00a0scrapy.Request(next_page,\u00a0callback=self.parse)<\/span><\/span><\/code><\/pre>\n<\/section>\n
\n
\u8fd9\u6837\u6211\u4eec\u5c31\u53ef\u4ee5\u83b7\u53d6\u5230\u6240\u6709\u9875\u9762\u7684\u6570\u636e\u4e86<\/p>\n
\n
\u63a5\u4e0b\u6765\u6211\u4eec\u8981\u628a\u6240\u6709\u7684\u6570\u636e\u4fdd\u5b58\u5230\u6570\u636e\u5e93<\/p>\n
\n
\u9996\u5148\u6211\u4eec\u5728 items.py \u4e2d\u5b9a\u4e49\u4e00\u4e0b\u6211\u4eec\u8981\u5b58\u50a8\u7684\u5b57\u6bb5<\/p>\n
\n
\n
import<\/span> scrapy\n<\/span><\/span><\/code>\n<\/span><\/code>class<\/span> QiushibaikeItem(scrapy.Item):\n<\/span><\/span><\/code>    # define the fields for your item here like:\n<\/span><\/span><\/code>    # name = scrapy.Field()\n<\/span><\/span><\/code>    author<\/span> = scrapy.Field()\n<\/span><\/span><\/code>    content<\/span> = scrapy.Field()\n<\/span><\/span><\/code>\u00a0\u00a0\u00a0\u00a0_id\u00a0<\/span>=\u00a0scrapy.Field()<\/span><\/span><\/code><\/pre>\n<\/section>\n
\n
\u63a5\u7740\u6211\u4eec\u5728 parse \u65b9\u6cd5\u4e2d\u5c06\u83b7\u53d6\u5230\u7684\u6570\u636e\u8d4b\u503c\u7ed9 item<\/p>\n
\n
\u5177\u4f53\u6765\u8bf4\u5c31\u662f\u8fd9\u6837<\/p>\n
\n
\n
    def<\/span> parse<\/span>(self, response)<\/span>:\n<\/span><\/span><\/code>\n<\/span><\/code>        content_left_div = response.xpath('\/\/*[@id=\"content-left\"]'<\/span>)\n<\/span><\/code>        content_list_div = content_left_div.xpath('.\/div'<\/span>)\n<\/span><\/code>\n<\/span><\/code>        for<\/span> content_div in<\/span> content_list_div:\n<\/span><\/code>            item = QiushibaikeItem()\n<\/span><\/code>            item['author'<\/span>] = content_div.xpath('.\/div\/a[2]\/h2\/text()'<\/span>).get()\n<\/span><\/code>            item['content'<\/span>] = content_div.xpath('.\/a\/div\/span\/text()'<\/span>).getall()\n<\/span><\/code>            item['_id'<\/span>] = content_div.attrib['id'<\/span>]\n<\/span><\/code>            yield<\/span> item<\/span><\/code>\n<\/span><\/code>        next_page = response.xpath('\/\/*[@id=\"content-left\"]\/ul\/li[last()]\/a'<\/span>).attrib['href'<\/span>]\n<\/span><\/code>\n<\/span><\/code>        if<\/span> next_page is<\/span> not<\/span> None<\/span>:\n<\/span><\/code>            yield<\/span> response.follow(next_page, callback=self.parse)<\/span><\/code><\/pre>\n<\/section>\n
\n
\u7b2c 7 \u884c\u5c31\u662f\u83b7\u53d6\u6211\u4eec\u521a\u521a\u5b9a\u4e49\u7684 item \u7684\u7c7b<\/p>\n
\n
8-10 \u884c\u5c31\u662f\u76f8\u5e94\u7684\u8d4b\u503c<\/p>\n
\n
\u90a3\u4e48\u6211\u4eec\u5b9a\u4e49\u597d\u4e86\u8981\u5b58\u50a8\u7684\u5b57\u6bb5\u4ee5\u53ca\u5199\u597d\u4e86\u6570\u636e\u722c\u53d6<\/p>\n
\n
\u63a5\u4e0b\u6765\u8fd8\u6709\u4e00\u6b65<\/p>\n
\n
\u5c31\u662f\u5b9a\u4e49\u597d\u6211\u4eec\u8981\u5b58\u50a8\u7684\u6570\u636e\u5e93<\/p>\n
\n
\u5230\u00a0pipelines.py \u4e2d<\/p>\n
\n
\n
class<\/span> QiushibaikePipeline<\/span>(object<\/span>):\n<\/span><\/span><\/code>\n<\/span><\/code>    def<\/span> __init__<\/span>(self<\/span>)<\/span><\/span>:\n<\/span><\/code>        self<\/span>.connection = pymongo.MongoClient('localhost'<\/span>, 27017<\/span>)\n<\/span><\/code>        self<\/span>.db = self<\/span>.connection.scrapy  \n<\/span><\/code>        self<\/span>.collection = self<\/span>.db.qiushibaike \n<\/span><\/code>\n<\/span><\/code>    def<\/span> process_item<\/span>(self<\/span>, item, spider)<\/span><\/span>:\n<\/span><\/code>        if<\/span> not<\/span> self<\/span>.connection or<\/span> not<\/span> item:\n<\/span><\/span><\/code>            return\n<\/span><\/span><\/code>        self<\/span>.collection.save(item)\n<\/span><\/code>\n<\/span><\/code>    def<\/span> __del__<\/span>(self<\/span>)<\/span><\/span>:\n<\/span><\/code>        if<\/span> self<\/span>.connection:\n<\/span><\/span><\/code>            self<\/span>.connection.close()<\/span><\/code><\/pre>\n<\/section>\n
\n
\u5728\u8fd9\u91cc\u6211\u4eec\u8fde\u63a5\u5230\u672c\u5730\u7684 MongoDB<\/p>\n
\n
\u5efa\u7acb\u4e86 scrapy \u6570\u636e\u5e93\u53ca\u4ee5\u4e0b\u7684\u00a0qiushibaike<\/p>\n
\n
\u63a5\u4e0b\u6765\u8fd8\u8981\u5728\u00a0settings.py \u6587\u4ef6\u4e2d\u914d\u7f6e\u4e0b<\/p>\n
\n
\n
# See https:\/\/doc.scrapy.org\/en\/latest\/topics\/item-pipeline.html\n<\/span><\/span><\/code>ITEM_PIPELINES = {\n<\/span><\/code>   'qiushibaike.pipelines.QiushibaikePipeline': 300,\n<\/span><\/code>}<\/span><\/code><\/pre>\n<\/section>\n
\n
\u8fd9\u6837\u624d\u53ef\u4ee5\u4f7f\u7528\u5230pipelines<\/span><\/p>\n
\u00a0<\/span><\/p>\n
\u5f53\u7136\u6211\u4eec\u8fd8\u53ef\u4ee5\u5728 settings.py \u91cc\u9762\u505a\u66f4\u591a\u7684\u8bbe\u7f6e<\/span><\/p>\n
\u00a0<\/span><\/p>\n
\u6bd4\u5982\u8bbe\u7f6e\u8bf7\u6c42\u5934<\/span><\/p>\n
\u00a0<\/span><\/p>\n
\n
# Crawl responsibly by identifying yourself (and your website) on the user-agent\n<\/span><\/span><\/code>USER_AGENT<\/span>\u00a0=\u00a0'Mozilla\/5.0\u00a0(X11;\u00a0Linux\u00a0x86_64)\u00a0AppleWebKit\/537.36\u00a0(KHTML,\u00a0like\u00a0Gecko)\u00a0Ubuntu\u00a0Chromium\/73.0.3683.86\u00a0Chrome\/73.0.3683.86\u00a0Safari\/537.36'<\/span><\/span><\/code><\/pre>\n<\/section>\n
\n
\n
ok<\/p>\n
\n
\u641e\u5b9a\u4e86\u4e4b\u540e<\/p>\n
\n
\u6211\u4eec\u4f7f\u7528\u547d\u4ee4\u6765\u6293\u53d6\u4e00\u4e0b<\/p>\n
\n
\n
scrapy<\/span>\u00a0crawl\u00a0qiushibaike<\/span><\/code><\/pre>\n<\/section>\n
\n
\u8fd0\u884c\u4e4b\u540e<\/p>\n
\n
\u6211\u4eec\u6253\u5f00 MongoDB \u770b\u770b<\/p>\n
\n
\"python\u722c\u866b30<\/p>\n
\n
\u53ef\u4ee5\u770b\u5230<\/p>\n
\n
\u6240\u6709\u7684\u6570\u636e\u5c31\u88ab\u6211\u722c\u53d6\u5230 MongoDB \u4e86<\/p>\n
\n
\"python\u722c\u866b30<\/p>\n
\n
ok<\/p>\n
\n
\u4ee5\u4e0a\u5c31\u662f scrapy \u7684\u7b80\u5355\u53c8\u725b\u903c\u7684\u64cd\u4f5c<\/p>\n
\n
\u66f4\u591a scrapy \u7684\u725b\u903c\u64cd\u4f5c\u53ef\u4ee5\u524d\u5f80\u00a0https:\/\/doc.scrapy.org\u00a0\u4e86\u89e3<\/p>\n
\n
\u8fd9\u4e24\u7bc7\u6240\u6d89\u53ca\u7684\u6e90\u4ee3\u7801\u5df2\u7ecf\u4e0a\u4f20<\/p>\n
\n
\u53ef\u4ee5\u5728\u516c\u4f17\u53f7\u540e\u53f0\u53d1\u9001\u00a0 scrapy<\/strong> \u83b7\u53d6<\/p>\n
\n
\u90a3\u4e48<\/p>\n
\n
\u6211\u4eec\u4e0b\u56de\u89c1<\/p>\n
\n
peace<\/span><\/p>\n
\n
\n
\n
\n
\"python\u722c\u866b30<\/p>\n
\n
\u5bf9\u4e86<\/p>\n
\n
\u6709\u4e2a\u4e8b<\/p>\n
\n
\u4f60\u5e0c\u671b\u63a5\u4e0b\u6765\u8fd9\u4e2a\u516c\u4f17\u53f7\u80fd\u6709\u66f4\u591a\u4ec0\u4e48\u6559\u7a0b<\/p>\n
\n
\u4f8b\u5982<\/p>\n
\n
\u722c\u866b\u5b9e\u6218\uff1f<\/p>\n
\n
ubuntu\uff1f<\/p>\n
\n
vim\uff1f<\/p>\n
\n
…?<\/p>\n
\n
\u8bf7\u626b\u4e0b\u65b9\u7684\u7801\u8bc4\u8bba\u544a\u8bc9\u6211\u4e00\u4e0b<\/p>\n
\n
\"python\u722c\u866b30<\/p>\n
\n
\n
\n
\n
\"python\u722c\u866b30<\/span><\/a><\/p>\n
\u539f\u6587\u59cb\u53d1\u4e8e\u5fae\u4fe1\u516c\u4f17\u53f7\uff08\u5b66\u4e60python\u7684\u6b63\u786e\u59ff\u52bf\uff09\uff1apython\u722c\u866b30 | scrapy\u540e\u7eed\uff0c\u628a\u300c\u7cd7\u4e8b\u767e\u79d1\u300d\u7684\u6bb5\u5b50\u722c\u4e0b\u6765\u7136\u540e\u5b58\u5230\u6570\u636e\u5e93\u4e2d<\/a><\/p><\/blockquote>\n<\/div>\n","protected":false},"excerpt":{"rendered":"
\u4e0a\u56de\u6211\u4eec\u8bf4\u5230 python\u722c\u866b29 | \u4f7f\u7528scrapy\u722c\u53d6\u7cd7\u4e8b\u767e\u79d1\u7684\u4f8b\u5b50\uff0c\u544a\u8bc9\u4f60\u5b83\u6709\u591a\u5389\u5bb3\uff01 WOW\uff01\uff01 scrapy awesome\uff01\uff01 \u600e\u4e48\u4f1a\u6709\u8fd9\u4e48\u725b\u903c\u7684\u6846\u67b6 wow\uff01\uff01 awesome\uff01\uff01 \u7528 scrapy \u6765\u722c\u53d6\u6570\u636e \u5c82\uff01\u4e0d\uff01\u662f\uff01\u975e\uff01\u5e38\uff01\u723d…<\/p>\n","protected":false},"author":1,"featured_media":5060,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[303],"tags":[401],"_links":{"self":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/5059"}],"collection":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=5059"}],"version-history":[{"count":2,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/5059\/revisions"}],"predecessor-version":[{"id":6643,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/5059\/revisions\/6643"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/media\/5060"}],"wp:attachment":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=5059"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=5059"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=5059"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}