Yleina

導(dǎo)航

公告

爬蟲作業(yè)

1.get()訪問百度主頁：

import requests


url = 'https://www.baidu.com'
for i in range(20):
    response = requests.get(url)
    print(f"第{i+1}次訪問")
    print(f'Response status: {response.status_code}')
    print(f'Text content length: {len(response.text)}')
    print(f'Content length: {len(response.content)}')
    print(response.text)

運行結(jié)果：

2.bs4庫練習(xí)使用：

from bs4 import BeautifulSoup
import re


text = """
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>菜鳥教程(runoob.com)</title>
</head>
<body>
<h1>我的第一個標題</h1>
<p id="first">我的第一個段落。</p>
</body>
<table border="1">
<tr>
<td>row 1, cell 1</td>
<td>row 1, cell 2</td>
</tr>
<tr>
<td>row 2, cell 1</td>
<td>row 2, cell 2</td>
</tr>
</table>
</html>
"""
# 創(chuàng)建BeautifulSoup對象
soup = BeautifulSoup(text, features="html.parser")

# 打印head標簽和學(xué)號后兩位
print(soup.head.prettify())
print("學(xué)號: 21\n")

# 獲取body標簽對象
print(soup.body.prettify())

# 獲取id為first的對象
first_p = soup.find(id="first")
print(first_p)

# 獲取打印中文字符
pattern = re.compile(u'[\u4e00-\u9fff]+')
chinese_chars = pattern.findall(text)
print(chinese_chars)

運行結(jié)果：

3.爬取中國大學(xué)排名(2020年)：

import requests
from bs4 import BeautifulSoup
import csv

all_univ = []


def get_html_text(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""


def fill_univ_list(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd) < 5:
            continue
        single_univ = [ltd[0].string.strip(), ltd[1].find('a', 'name-cn').string.strip(), ltd[2].text.strip(),
                       ltd[4].string.strip()]
        all_univ.append(single_univ)


def print_univ_list(num):
    file_name = "大學(xué)排行.csv"
    print("{0:^10}\t{1:{4}^10}\t{2:^10}\t{3:^10}".format("排名", "學(xué)校名稱", "省市", "總分", chr(12288)))
    with open(file_name, 'w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(["排名", "學(xué)校名稱", "省市", "總分"])
        for i in range(num):
            u = all_univ[i]
            writer.writerow(u)
            print("{0:^10}\t{1:{4}^10}\t{2:^10}\t{3:^10}".format(u[0], u[1], u[2], u[3], chr(12288)))


def main(num):
    url = "https://www.shanghairanking.cn/rankings/bcur/202011.html"
    html = get_html_text(url)
    soup = BeautifulSoup(html, features="html.parser")
    fill_univ_list(soup)
    print_univ_list(num)


main(20)

運行結(jié)果：

csv文件內(nèi)容

posted on 2023-12-11 22:18 伊蕾娜。閱讀(26) 評論(0) 收藏舉報

刷新頁面返回頂部