使用python從網站下載文件

Question

我需要從（https://www.sec.gov/litigation/suspensions.shtml）給定網站下載所有文件。它有從1995年到2017年的數據，每年裏面有多個需要下載的文件的鏈接。 Th文件使用.pdf，.htm和.txt格式。我試圖通過查看各種教程來抓取數據，但是我需要做的與通常的網頁抓取教程不同。我使用了下面的代碼，但它並沒有達到我的目的。我是python的新手，我在這裏被困在如何前進的道路上。任何人都可以請建議需要做什麼。

import requests 
from bs4 import BeautifulSoup 

r = requests.get("https://www.sec.gov/litigation/suspensions.shtml") 
r.content 

soup = BeautifulSoup(r.content) 
print soup.prettify() 

links = soup.find_all("a") 

for link in links: 
     print "<a href= '%s'>%s</a>" %(link.get("href"), link.text) 

g_data = soup.find_all("p", {"id": "archive-links"}) 
print g_data 

for item in g_data: 
    print item.text 

Answer 1

這應該做的工作。檢查了Python 3.6，但代碼應該是Python2.7兼容的。 主要想法是找到每年的鏈接，然後抓取每年的pdf，htm和txt文件的所有鏈接。

from __future__ import print_function 

import requests 
from bs4 import BeautifulSoup 


def file_links_filter(tag): 
    """ 
    Tags filter. Return True for links that ends with 'pdf', 'htm' or 'txt' 
    """ 
    if isinstance(tag, str): 
     return tag.endswith('pdf') or tag.endswith('htm') or tag.endswith('txt') 


def get_links(tags_list): 
    return [WEB_ROOT + tag.attrs['href'] for tag in tags_list] 


def download_file(file_link, folder): 
    file = requests.get(file_link).content 
    name = file_link.split('/')[-1] 
    save_path = folder + name 

    print("Saving file:", save_path) 
    with open(save_path, 'wb') as fp: 
     fp.write(file) 


WEB_ROOT = 'https://www.sec.gov' 
SAVE_FOLDER = '~/download_files/' # directory in which files will be downloaded 

r = requests.get("https://www.sec.gov/litigation/suspensions.shtml") 

soup = BeautifulSoup(r.content, 'html.parser') 

years = soup.select("p#archive-links > a") # css selector for all <a> inside <p id='archive'> tag 
years_links = get_links(years) 

links_to_download = [] 
for year_link in years_links: 
    page = requests.get(year_link) 
    beautiful_page = BeautifulSoup(page.content, 'html.parser') 

    links = beautiful_page.find_all("a", href=file_links_filter) 
    links = get_links(links) 

    links_to_download.extend(links) 

# make set to exclude duplicate links 
links_to_download = set(links_to_download) 

print("Got links:", links_to_download) 

for link in set(links_to_download): 
    download_file(link, SAVE_FOLDER)