抓取百度博客文章的Python脚本

1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

import urllib2
import urllib
import httplib
import cookielib
import os
import re

import socket  

#下载博客，其实，最好用多线程＋wget之类的做，省事
def downloadblog(path = 'yobin',url_hi='http://hi.baidu.com/yobin/',blogUrl = '5383d42a45378d2bd42af18b.html'):
    #http://hi.baidu.com/yobin/blog/item/5383d42a45378d2bd42af18b.html
    url = url_hi + 'blog/item/' + blogUrl
    print url
    nFail = 0
    while nFail < 5:
     try:
            sock = urllib.urlopen(url)
            htmlSource = sock.read()
            path += '/%s' % (blogUrl)
            myfile = file(path,'w')#文件名是****.html
            myfile.write(htmlSource)
            myfile.close() 
            sock.close()
            return
     except:
      nFail += 1
    print 'download blog fail:%s' % (blogUrl)

def getbaidublog(url_hi='http://hi.baidu.com/yobin/',username = 'yyobin',password = ''):
    socket.setdefaulttimeout(30)  
    if not os.path.exists(username):
     os.mkdir(username)
         
    #login baidu
    #-------------------------------------------------
    cj = cookielib.CookieJar()
    post_data = urllib.urlencode({'username':username,
                                  'password':password,
                                  'pwd':'1'})
    path = 'http://passport.baidu.com/?login'
   
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    opener.addheaders = [('User-agent','Opera/9.23')]
    urllib2.install_opener(opener)
    req = urllib2.Request(path,post_data)
    conn = urllib2.urlopen(req)
    #-------------------------------------------------

    #-------------------------------------------------
    index = 0
    totalblogs = 0
    html = '<h1><a href="%s">Home</a></h1><Br/>' % (url_hi)

    while 1:
        url_blog = url_hi + 'blog/index/%d' % index
        print url_blog
        req2 = urllib2.Request(url_blog)
        content = urllib2.urlopen(req2).read()
        
        if content:
            #解析一个index页，获取博客连接，时间太久远，似乎爬不下来了
            matched_objs = re.findall('''<div class="tit"><a href="/.*/blog/item/(.*)" target="_blank">(.*)</a></div>\r\n\t<div class="date">(.*)</div>''', content)
            if len(matched_objs) == 0:
                print "The end"
                break
            #下载每个链接，这里最好用多线程来写，这样会快很多
            for matched_obj in matched_objs:
                totalblogs += 1
                blogUrl,blogTitle,date = matched_obj
                html += '<th><a href="%s" target="_blank">%s</a></th><th>    </th><th>%s</th><Br/>' % (blogUrl,blogTitle,date)
                downloadblog(username,url_hi,blogUrl)#下载博客，其实，最好用多线程＋wget之类的做，省事
        index += 1
        time.sleep(5)
        
    html += 'Total blog:%d' % (totalblogs)
    path = '%s/index.html' % (username)
    myfile = file(path,'w')
    myfile.write(html)
    myfile.close()
    return

if __name__ == "__main__":
    url_hi='http://hi.baidu.com/yobin/'
    username = 'yyobin'
    password = ''#输入密码，主要是可以看到有权限才能看的文章
    getbaidublog(url_hi,username,password)