python爬取百度圖片

# 定義一個(gè)getHtml()函數(shù)
def getHtml(url):
    try:
        page = urllib.urlopen(url)  # urllib.urlopen()方法用于打開一個(gè)URL地址
        html = page.read()  # read()方法用于讀取URL上的數(shù)據(jù)
    except Exception as e:
        html = ''
    return html

# 得到下一頁的url
def get_page_url(html):
    url_reg = r'<a href="(.*)">下一頁</a>'
    url_pattern = re.compile(url_reg)
    fanye_urls = url_pattern.findall(html)
    fanye_url = 'https://tieba.baidu.com' + fanye_urls[0] if fanye_urls else ''
    return fanye_url

def getImg(html):
    img_reg = r'https://.[^\s]+?.jpg|https://.[^\s]+?.png'  # 正則表達(dá)式，得到圖片地址
    img_pattern = re.compile(img_reg)  # re.compile() 可以把正則表達(dá)式編譯成一個(gè)正則表達(dá)式對(duì)象.
    imgList = img_pattern.findall(html)  # img_pattern.findall() 方法讀取html 中包含 img_reg（正則表達(dá)式）的  數(shù)據(jù)，數(shù)組形式
    return imgList

def down_img(imgList):
    x = 0
    for imgUrl in imgList:
        try:
            # 核心是urllib.urlretrieve()方法,直接將遠(yuǎn)程數(shù)據(jù)下載到本地，圖片通過x依次遞增命名
            urllib.urlretrieve(imgUrl, 'E:\img\%s.jpg' % str(x + 1))
            print '成功下載第%s張圖片時(shí)：%s' % (str(x + 1), str(imgUrl))
            x += 1
        except Exception as e:
            print '下載第%s張圖片時(shí)失敗：%s' % (str(x + 1), str(imgUrl))
            print e
            continue
    return imgList

all_img_urls = []
    # 得到網(wǎng)頁源碼
    html = getHtml("https://tieba.baidu.com/p/5407739329")
    fanye_url = get_page_url(html)
    # 得到圖片鏈接的數(shù)組
    imgList = getImg(html)
    # 將imgList數(shù)組存入總的圖片數(shù)組內(nèi)
    all_img_urls.extend(imgList)
    fanye_count = 0  # 累計(jì)翻頁數(shù)
    while 1:
        try:
            next_html = getHtml(fanye_url)
            fanye_url = get_page_url(next_html)
            next_imgList = getImg(next_html)
            fanye_count += 1
            print('第%s頁' % fanye_count)
            all_img_urls.extend(next_imgList)
            if fanye_url == '' and next_imgList == []:
                print('已到最后一頁, 開始下載：')
                break
        except Exception as e:
            print e
            continue
    down_img(all_img_urls)

# coding=utf-8
"""下載百度貼吧內(nèi)的圖片"""
import re
import urllib


# 定義一個(gè)getHtml()函數(shù)
def getHtml(url):
    try:
        page = urllib.urlopen(url)  # urllib.urlopen()方法用于打開一個(gè)URL地址
        html = page.read()  # read()方法用于讀取URL上的數(shù)據(jù)
    except Exception as e:
        html = ''
    return html


# 得到下一頁的url
def get_page_url(html):
    url_reg = r'<a href="(.*)">下一頁</a>'
    url_pattern = re.compile(url_reg)
    fanye_urls = url_pattern.findall(html)
    fanye_url = 'https://tieba.baidu.com' + fanye_urls[0] if fanye_urls else ''
    return fanye_url


def getImg(html):
    img_reg = r'https://.[^\s]+?.jpg|https://.[^\s]+?.png'  # 正則表達(dá)式，得到圖片地址
    img_pattern = re.compile(img_reg)  # re.compile() 可以把正則表達(dá)式編譯成一個(gè)正則表達(dá)式對(duì)象.
    imgList = img_pattern.findall(html)  # img_pattern.findall() 方法讀取html 中包含 img_reg（正則表達(dá)式）的  數(shù)據(jù)，數(shù)組形式
    return imgList


def down_img(imgList):
    x = 0
    for imgUrl in imgList:
        try:
            # 核心是urllib.urlretrieve()方法,直接將遠(yuǎn)程數(shù)據(jù)下載到本地，圖片通過x依次遞增命名
            urllib.urlretrieve(imgUrl, 'E:\img\%s.jpg' % str(x + 1))
            print '成功下載第%s張圖片時(shí)：%s' % (str(x + 1), str(imgUrl))
            x += 1
        except Exception as e:
            print '下載第%s張圖片時(shí)失敗：%s' % (str(x + 1), str(imgUrl))
            print e
            continue
    return imgList


if __name__ == '__main__':
    all_img_urls = []
    # 得到網(wǎng)頁源碼
    html = getHtml("https://tieba.baidu.com/p/5407739329")
    fanye_url = get_page_url(html)
    # 得到圖片鏈接的數(shù)組
    imgList = getImg(html)
    # 將imgList數(shù)組存入總的圖片數(shù)組內(nèi)
    all_img_urls.extend(imgList)
    fanye_count = 0  # 累計(jì)翻頁數(shù)
    while 1:
        try:
            next_html = getHtml(fanye_url)
            fanye_url = get_page_url(next_html)
            next_imgList = getImg(next_html)
            fanye_count += 1
            print('第%s頁' % fanye_count)
            all_img_urls.extend(next_imgList)
            if fanye_url == '' and next_imgList == []:
                print('已到最后一頁, 開始下載：')
                break
        except Exception as e:
            print e
            continue
    down_img(all_img_urls)