python-待更新

1.基础版

import requests
from bs4 import BeautifulSoup
allUniv = []
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd) == 0:
            continue
        singleUniv = [td.text.strip() for td in ltd]
        allUniv.append(singleUniv)
def printUnivList(num):
    print("{:^8}{:^32}{:^8}{:^8}{:^8}{:^32}".format("排名", "学校名称","省市", "类型", "总分","办学层次"))
    for i in range(num):
        u = allUniv[i]
        print("{:^8}{:^32}{:^8}{:^8}{:^8}{:^32}".format(u[0], u[1], u[2], u[3],u[4],u[5]))

def write_to_txt(data, filename='university_ranking.txt'):
    # 打开文件以写入模式
    with open(filename, 'w', encoding='utf-8') as f:
        # 写入表头
        headers = ["排名", "学校名称", "省市", "类型", "总分", "办学层次"]
        f.write('\t'.join(headers) + '\n')  # 使用制表符作为分隔符
        # 写入数据
        for univ in data:
            f.write('\t'.join(univ) + '\n')  # 使用制表符作为分隔符

    print(f"数据已成功写入文本文件：{filename}")
def main():
    url = 'https://www.shanghairanking.cn/rankings/bcur/2024'
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    fillUnivList(soup)
    printUnivList(30)
    write_to_txt(allUniv)  # 调用write函数，并传入allUniv列表

if __name__ == "__main__":
    main()

2.升级版

from bs4 import BeautifulSoup
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
# 配置Chrome浏览器选项  
option = webdriver.ChromeOptions()
option.add_experimental_option("detach", True)  # 让浏览器在后台运行  
# 定义URL  
url = "https://www.shanghairanking.cn/rankings/bcur/2024"
# 初始化浏览器并访问URL  
browser = webdriver.Chrome(options=option)
browser.get(url)
# 初始化列表来存储所有页面的数据  
all_data = []
def getData():
    global all_data  # 使用全局变量来存储数据  
    html = browser.page_source  # 获取当前页面的HTML  
    soup = BeautifulSoup(html, "html.parser")
    tbody = soup.find('tbody')  # 查找tbody元素  
    if tbody:  # 确保tbody存在  
        for tr in tbody.find_all('tr'):  # 查找所有的tr元素  
            if tr.find('td'):  # 确保tr元素包含td元素  
                tds = tr.find_all('td')  # 查找所有的td元素  
                # 提取文本并去除空格，然后将它们添加到all_data列表中  
                all_data.append([td.get_text(strip=True) for td in tds])
            # 爬取数据的函数
def crawl_pages(max_pages):
    current_page = 1  # 当前页码初始化为1  
    while current_page <= max_pages:  # 循环爬取指定数量的页面  
        getData()  # 获取当前页面的数据  
        time.sleep(3)  # 等待3秒  
        # 检查是否有下一页按钮，并点击它（如果有的话）  
        try:
            next_page = browser.find_element(By.CSS_SELECTOR, 'li.ant-pagination-next>a')
            next_page.click()
            current_page += 1  # 成功点击后，增加当前页码  
        except:
            break  # 如果没有下一页按钮，则跳出循环  
# 爬取20页数据
crawl_pages(20)
# 将数据存储到txt文件
def write_to_txt(data, filename='中国大学排名.txt'):
    with open(filename, 'w', encoding='utf-8') as f:
        headers = ["排名", "学校名称", "省市", "类型", "总分"]  # 根据实际网页内容调整表头  
        f.write('\t'.join(headers) + '\n')  # 写入表头  
        for row in data:
            f.write('\t'.join(row) + '\n')  # 写入每一行数据  
# 调用函数将数据写入txt文件
write_to_txt(all_data)
# 关闭浏览器  
browser.quit()

university_ranking.txt

5-16

5-16-龙圆芝.rar

5-23

5-23-龙圆芝

附件1：123家有信贷记录企业的相关数据 附件2：302家无信贷记录企业的相关数据 附件3：银行贷款年利率与客户流失率关系的统计数据 中小微企业的信贷决策