{"id":3496,"date":"2019-07-28T00:40:50","date_gmt":"2020-02-12T14:10:46","guid":{"rendered":"http:\/\/vip.fxxkpython.com\/?p=3496"},"modified":"2020-02-12T22:10:49","modified_gmt":"2020-02-12T14:10:49","slug":"adfa65fc6c","status":"publish","type":"post","link":"https:\/\/vip.fxxkpython.com\/?p=3496","title":{"rendered":"\u7ed9\u4f60\u4eec\u8bf4\u51e0\u70b9\u9c9c\u6709\u4eba\u77e5\u7684\u722c\u866b\u6280\u5de7"},"content":{"rendered":"
\n


<\/p>\n

\u4eca\u5929\u5c0f\u5e05b\u60f3\u8ddf\u5206\u4eab\u51e0\u4e2a<\/span>
<\/p>\n

\u5173\u4e8e\u722c\u866b\u53ef\u4ee5\u7528\u5230\u7684\u6280\u5de7<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n


<\/p>\n

\u53ea\u8981\u6280\u5de7\u4f7f\u7528\u5f97\u5f53<\/span><\/p>\n

\u90a3\u4e48\u5c3d\u7ba1\u59ff\u52bf\u518d\u591a
<\/span><\/p>\n

\u4e5f\u80fd\u611f\u5230\u723d\u723d\u7684\u4e86
<\/span><\/p>\n


<\/span><\/p>\n

\u90a3\u4e48\uff0c\u5982\u4f55\u624d\u80fd\u723d\u5462\uff1f<\/span><\/p>\n

\u63a5\u4e0b\u6765\u5c31\u662f<\/span><\/p>\n

\u5b66\u4e60python\u7684\u6b63\u786e\u59ff\u52bf<\/span><\/strong><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/a><\/p>\n

<\/span>
<\/p>\n

\u6280\u5de7\u4e00
<\/span><\/p>\n

\u6362\u4e2a\u89d2\u5ea6\uff0c\u89e3\u9501\u65b0\u59ff\u52bf<\/span><\/strong><\/p>\n


<\/p>\n

\u6211\u77e5\u9053<\/span><\/p>\n

\u4f60\u5728\u722c\u53d6\u67d0\u4e9b web \u7f51\u7ad9\u7684\u65f6\u5019<\/span><\/p>\n

\u88ab\u5404\u79cd\u53cd\u722c\u5f04\u5f97\u54ed\u5929\u558a\u5730<\/span><\/p>\n


<\/span><\/p>\n

\u4ec0\u4e48\u51e0\u628a css \u5b57\u4f53\u52a0\u5bc6<\/span><\/p>\n

\u4ec0\u4e48\u51e0\u628a js \u7684 MD5 \u7b49<\/span><\/p>\n

\u5404\u79cd\u4e71\u4e03\u516b\u7cdf\u7684\u52a0\u5bc6<\/span><\/p>\n

\u4ec0\u4e48\u51e0\u628a\u5404\u79cd\u98de\u7684\u9a8c\u8bc1\u7801<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n


<\/p>\n

\u8fd9\u65f6\u5019<\/span><\/p>\n

\u5c31\u522b\u592a\u6267\u7740\u4e8e web PC \u7aef\u561b<\/span><\/p>\n

\u54b1\u4eec\u53bb\u770b\u770b\u4eba\u5bb6\u7684\u79fb\u52a8\u7aef
<\/span><\/p>\n

\u770b\u770b\u4eba\u5bb6\u7684 H5<\/span><\/p>\n


<\/span><\/p>\n

\u5728\u722c\u53d6\u524d\u53ef\u4ee5\u95ee\u95ee\u5bf9\u65b9\uff1a<\/span><\/p>\n

\u201c\u5728\u5417\uff1f\u770b\u770b H5\u201d<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n

\u53ef\u80fd\u4f60\u4f1a\u5728\u79fb\u52a8\u7aef\u53d1\u73b0\u60ca\u559c
<\/span><\/p>\n

\u6570\u636e\u90fd\u662f\u4e00\u6837\u7684\u6570\u636e<\/span><\/p>\n

\u51a4\u51a4\u76f8\u62a5\u4f55\u65f6\u4e86<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/span><\/p>\n

\u6280\u5de7\u4e8c
<\/span><\/p>\n

\u591c\u592a\u7f8e\uff0c\u722c\u866b\u5c31\u6ca1\u90a3\u4e48\u5371\u9669<\/span><\/strong><\/p>\n


\u5728\u722c\u53d6\u7684\u65f6\u5019<\/span>
<\/p>\n

\u4e0d\u8981\u731b\u653b\u561b\uff5e<\/span><\/p>\n

\u4eba\u5bb6\u53d7\u4e0d\u4e86\u554a<\/span><\/p>\n


<\/span><\/p>\n

\u4f60\u8981\u5b66\u4f1a\u505c\u987f
<\/span><\/p>\n

\u514b\u5236\u4e00\u70b9
<\/span><\/p>\n

\u8be5 sleep \u5c31 sleep
<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\u8981\u8d81\u4eba\u5bb6\u7761\u89c9\u7684\u65f6\u5019<\/span><\/p>\n

\u9650\u5236\u9632\u8303\u7a0b\u5ea6\u4f4e\u7684\u65f6\u5019
<\/span><\/p>\n

\u80fd\u665a\u70b9\u5c31\u665a\u70b9\u518d\u53bb\u722c<\/span><\/p>\n

\u6ca1\u770b\u8fc7\u51cc\u6668\u56db\u70b9\u7684\u6d1b\u6749\u77f6<\/span><\/p>\n

\u4f46\u662f\u4f60\u53ef\u4ee5\u770b\u5230\u51cc\u6668\u56db\u70b9\u7684\u722c\u866b\u554a<\/span><\/p>\n


<\/span><\/p>\n

\u8fd9\u6837\u4f60\u4e70\u7684 IP \u624d\u4e0d\u4f1a\u9891\u7e41\u88ab\u5c01<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/p>\n

\u6280\u5de7\u4e09
<\/span><\/p>\n

\u5584\u7528\u4ed6\u4eba\u7684 UA<\/span><\/strong><\/p>\n


<\/span><\/p>\n

\u5982\u679c\u4f60\u53bb\u770b\u522b\u4eba\u7f51\u7ad9\u7684 robots.txt<\/span><\/p>\n

\u4f60\u5c31\u4f1a\u770b\u5230\u522b\u4eba\u7684\u58f0\u660e<\/span><\/p>\n


<\/span><\/p>\n

\u58f0\u660e\u4ec0\u4e48\u4e1c\u897f\u662f\u53ef\u4ee5\u722c\u53d6
<\/span><\/p>\n

\u4ec0\u4e48\u4e1c\u897f\u662f\u4e0d\u5141\u8bb8\u88ab\u722c\u7684<\/span><\/p>\n


<\/span><\/p>\n

\u4f46\u4f60\u5e38\u5e38\u5ffd\u7565\u4e86\u4e00\u4e2a\u4e1c\u897f<\/span><\/p>\n

\u4eba\u5bb6\u58f0\u660e\u4e86\u5e0c\u671b\u7ed9\u4ec0\u4e48\u641c\u7d22\u5f15\u64ce\u722c<\/span><\/p>\n


<\/span><\/p>\n

\u6bd4\u5982\u8fd9\u4e2a<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/p>\n

<\/span>
<\/p>\n


<\/p>\n

\u770b\u5230\u6ca1
<\/span><\/p>\n

\u8fd9\u662f\u522b\u4eba\u5b9a\u4e49\u7684 robots
<\/span><\/p>\n

\u503c\u5f97\u6ce8\u610f\u7684\u662f<\/span><\/p>\n

\u5c1a\u4eae\u4eae\u7684 <\/span>User-agent<\/span><\/strong>
<\/span><\/p>\n


<\/span><\/p>\n

\u90a3\u4e48\u5f53\u4f60\u5728 Python \u6784\u9020 Header \u7684\u65f6\u5019
<\/span><\/p>\n

User-agent \u5c31\u76f4\u63a5\u6307\u5b9a\u4ed6\u4eec robots \u5b9a\u4e49\u7684\u5c31\u597d\u4e86\u554a
<\/span><\/p>\n


<\/span><\/p>\n

\u6bd4\u5982 \u767e\u5ea6\u7684UA\uff0cgoogle\u7684UA\uff0c360\u7684UA<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\u4f60\u518d\u53bb\u722c\u53d6\u770b\u770b<\/span><\/p>\n

\u90a3\u662f\u4e00\u4e2a\u53cb\u597d\u554a<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/span><\/p>\n

\u6280\u5de7\u56db
<\/span><\/p>\n

\u63d2\u4ef6\u8ba9\u4f60\u8282\u7701\u65f6\u95f4<\/span><\/strong><\/p>\n


<\/p>\n

\u6709\u65f6\u5019\u6211\u4eec\u8981\u62ff\u4e00\u4e9b\u5173\u952e\u7684\u6570\u636e<\/span><\/p>\n

\u5f80\u5f80\u4f1a\u7528\u5230<\/span> xpath\u3001css selctor \u4e4b\u7c7b\u7684<\/p>\n


<\/p>\n

\u81ea\u5df1\u4e00\u4e2a\u4e00\u4e2a\u53bb\u6bd4\u5bf9\u83b7\u53d6<\/span>
<\/p>\n

\u90a3\u5c31\u592a\u9ebb\u70e6\u4e86\u554a<\/span><\/p>\n


<\/span><\/p>\n

\u8fd8\u8bb0\u5f97\u4e4b\u524d\u8bf4\u5f97\u8fd9\u4e2a\u5417\uff1f<\/span><\/p>\n

\u5410\u8840\u5206\u4eab\u8fd9\u4e24\u4e2a\u722c\u866b\u7528\u5230\u7684 Chrome \u725b\u903c\u63d2\u4ef6<\/a>
<\/span><\/p>\n


<\/span><\/p>\n

\u8bb0\u5f97\u7528\u8d77\u6765\u5440
<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/p>\n

\u6280\u5de7\u4e94
<\/span><\/p>\n

\u90a3 Header \u5feb\u901f\u751f\u6210\u5427<\/span><\/strong><\/p>\n


<\/p>\n

\u6bcf\u4e00\u6b21\u4f60\u5728\u590d\u5236 request header \u7684\u65f6\u5019<\/span><\/p>\n

\u662f\u4e0d\u662f\u6709\u4e00\u5927\u4e32\u6709\u7684\u6ca1\u7684<\/span><\/p>\n

\u53c8\u4e0d\u5f97\u4e0d\u590d\u5236\u8fc7\u6765<\/span><\/p>\n

\u5728\u4f60\u7684 Python \u4e2d\u4f7f\u7528<\/span><\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/span><\/p>\n


<\/span><\/p>\n

\u53ef\u662f<\/span><\/p>\n

\u683c\u5f0f\u53c8\u4e0d\u5bf9
<\/span><\/p>\n

\u6bcf\u6b21\u64cd\u4f5c\u5f88\u9ebb\u70e6\u662f\u4e0d\uff1f<\/span><\/p>\n


<\/span><\/p>\n

\u90a3\u4f60\u53ef\u4ee5\u81ea\u5df1\u5199\u4e00\u4e2a\u65b9\u6cd5<\/span><\/p>\n

\u53c2\u6570\u5c31\u662f\u4f60\u590d\u5236\u7684 header \u5b57\u7b26\u4e32<\/span><\/p>\n

\u7136\u540e\u751f\u6210 header \u7684\u5b57\u5178\u683c\u5f0f<\/span><\/p>\n


<\/span><\/p>\n

\u4e0d\u5c31\u5b8c\u4e8b\u4e86<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/p>\n

\u6280\u5de7\u516d
<\/span><\/p>\n

\u722c\u53d6\u6574\u7ad9\u5176\u5b9e\u662f\u8fd9\u6837\u7684<\/span><\/strong><\/p>\n


<\/p>\n

\u6709\u65f6\u5019\u4f60\u60f3\u722c\u53d6\u6574\u4e2a\u7f51\u7ad9\u7684url<\/span><\/p>\n

\u600e\u4e48\u529e\u5462\uff1f<\/span><\/p>\n


<\/span><\/p>\n

\u4e0d\u662f\u53bb\u9996\u9875\u4e00\u4e2a\u4e00\u4e2a\u6293<\/span><\/p>\n

\u4f60\u5e94\u8be5\u627e\u5230\u5bf9\u65b9\u7684 sitemap.xml<\/span><\/p>\n


<\/span><\/p>\n

\u56e0\u4e3a\u7f51\u7ad9\u4e00\u822c\u5e0c\u671b Google \u6216\u8005\u767e\u5ea6\u5feb\u70b9\u6536\u5f55\u4ed6\u4eec\u7684\u7f51\u7ad9<\/span><\/p>\n


<\/p>\n

\u6240\u4ee5\u4ed6\u4eec\u4f1a\u628a\u81ea\u5df1\u7684\u7f51\u7ad9\u7684 url \u751f\u6210 sitemap \u63d0\u4ea4<\/span><\/p>\n


<\/span><\/p>\n

\u8fd9\u4e2a\u65f6\u5019 sitemap \u5c31\u5305\u542b\u4e86\u8fd9\u4e2a\u7f51\u7ad9\u6240\u6709\u53ef\u722c\u53d6\u7684 url<\/span><\/p>\n


<\/p>\n

sitemap\u4e00\u822c\u5728\u7f51\u7ad9\u7684\u6839\u76ee\u5f55\u4e0b<\/span><\/p>\n

\u53ef\u4ee5\u5728\u4ed6\u4eec\u7684 robots.txt \u770b\u770b\u4ed6\u4eec\u6307\u5b9a\u7684\u4f4d\u7f6e<\/span><\/p>\n


<\/span><\/p>\n

\u6bd4\u5982\u732b\u773c\u7535\u5f71\u7684sitemap\uff1a<\/span><\/p>\n

<\/span><\/p>\n


<\/span><\/p>\n

\"python\"<\/p>\n

<\/span>
<\/p>\n


<\/span><\/p>\n


<\/span><\/p>\n

\u4ece\u800c\u83b7\u53d6 sitemap \u518d\u53bb\u8bf7\u6c42\u91cc\u9762\u7684 url \u5373\u53ef\uff01<\/span><\/p>\n


<\/span><\/p>\n

\n

<\/span>
<\/p>\n


<\/p>\n

ojbk<\/span>
<\/p>\n

\u4ee5\u4e0a\u5c31\u662f\u5c0f\u5e05b<\/span><\/p>\n

\u7ed9\u4f60\u5206\u4eab\u7684\u722c\u866b\u6280\u5de7<\/span><\/p>\n

\u5e0c\u671b\u5bf9\u4f60\u6709\u5e2e\u52a9<\/span><\/p>\n

\u90a3\u4e48\u6211\u4eec\u4e0b\u56de\u89c1<\/span><\/p>\n

peace<\/span><\/strong><\/p>\n


<\/span><\/strong><\/p>\n

\"python\"<\/p>\n

\u626b\u4e00\u626b<\/span><\/strong><\/p>\n

\u5b66\u4e60 Python \u6ca1\u70e6\u607c<\/span><\/strong><\/p>\n


<\/span><\/strong><\/p>\n

\"python\"<\/p>\n


<\/span><\/strong><\/p>\n

\n

\u539f\u6587\u59cb\u53d1\u4e8e\u5fae\u4fe1\u516c\u4f17\u53f7\uff08\u5b66\u4e60python\u7684\u6b63\u786e\u59ff\u52bf\uff09\uff1apython<\/a><\/p>\n<\/blockquote>\n<\/div>\n","protected":false},"excerpt":{"rendered":"

\u4eca\u5929\u5c0f\u5e05b\u60f3\u8ddf\u5206\u4eab\u51e0\u4e2a \u5173\u4e8e\u722c\u866b\u53ef\u4ee5\u7528\u5230\u7684\u6280\u5de7 \u53ea\u8981\u6280\u5de7\u4f7f\u7528\u5f97\u5f53 \u90a3\u4e48\u5c3d\u7ba1\u59ff\u52bf\u518d\u591a \u4e5f\u80fd\u611f\u5230\u723d\u723d\u7684\u4e86 \u90a3\u4e48\uff0c\u5982\u4f55\u624d\u80fd\u723d\u5462\uff1f \u63a5\u4e0b\u6765\u5c31\u662f \u5b66\u4e60python\u7684\u6b63\u786e\u59ff\u52bf \u6280\u5de7\u4e00 \u6362\u4e2a\u89d2\u5ea6\uff0c\u89e3\u9501\u65b0\u59ff\u52bf \u6211\u77e5\u9053 \u4f60\u5728\u722c\u53d6\u67d0\u4e9b web \u7f51\u7ad9\u7684\u65f6\u5019 \u88ab\u5404\u79cd\u53cd\u722c\u5f04\u5f97…<\/p>\n","protected":false},"author":1,"featured_media":3497,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[303],"tags":[302,19,300,301],"_links":{"self":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/3496"}],"collection":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3496"}],"version-history":[{"count":1,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/3496\/revisions"}],"predecessor-version":[{"id":3509,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/posts\/3496\/revisions\/3509"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=\/wp\/v2\/media\/3497"}],"wp:attachment":[{"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3496"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3496"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/vip.fxxkpython.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3496"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}