PythonでのWebスクレイピング：Requests、BeautifulSoup、Seleniumを使った完全ガイド

import requests
from bs4 import BeautifulSoup
 
# ページを取得
url = 'https://books.toscrape.com/'
response = requests.get(url)
response.raise_for_status()  # エラーステータスコードで例外を発生
 
# HTMLを解析
soup = BeautifulSoup(response.text, 'html.parser')
 
# 書名を抽出
books = soup.select('article.product_pod h3 a')
for book in books[:5]:
    print(book['title'])

# 必要なパッケージをインストール
# pip install requests beautifulsoup4 lxml

from bs4 import BeautifulSoup
 
html = """
<div class="products">
    <div class="product" id="p1">
        <h2 class="name">Widget A</h2>
        <span class="price">$9.99</span>
        <p class="desc">A useful widget</p>
    </div>
    <div class="product" id="p2">
        <h2 class="name">Widget B</h2>
        <span class="price">$14.99</span>
        <p class="desc">A better widget</p>
    </div>
</div>
"""
 
soup = BeautifulSoup(html, 'html.parser')
 
# タグで検索
print(soup.find('h2').text)  # "Widget A"
 
# クラスで検索
prices = soup.find_all('span', class_='price')
for p in prices:
    print(p.text)  # "$9.99", "$14.99"
 
# IDで検索
product = soup.find(id='p2')
print(product.find('h2').text)  # "Widget B"
 
# CSSセレクタ
names = soup.select('.product .name')
for n in names:
    print(n.text)

from bs4 import BeautifulSoup
 
html = '<a href="/page/2" class="next" data-page="2">Next Page</a>'
soup = BeautifulSoup(html, 'html.parser')
link = soup.find('a')
 
# テキストコンテンツ
print(link.text)           # "Next Page"
print(link.get_text(strip=True))  # "Next Page"（空白除去）
 
# 属性
print(link['href'])        # "/page/2"
print(link.get('class'))   # ['next']
print(link['data-page'])   # "2"

import requests
from bs4 import BeautifulSoup
 
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
}
 
# セッションを使用してcookieと永続的なヘッダーを管理
session = requests.Session()
session.headers.update(headers)
 
response = session.get('https://books.toscrape.com/')
soup = BeautifulSoup(response.text, 'html.parser')
print(f"{len(soup.select('article.product_pod'))}冊の本を発見")

import requests
from bs4 import BeautifulSoup
 
base_url = 'https://books.toscrape.com/catalogue/'
all_books = []
url = 'https://books.toscrape.com/catalogue/page-1.html'
 
while url:
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
 
    # 現在のページから書籍を抽出
    for book in soup.select('article.product_pod'):
        title = book.select_one('h3 a')['title']
        price = book.select_one('.price_color').text
        all_books.append({'title': title, 'price': price})
 
    # 次のページを探す
    next_btn = soup.select_one('li.next a')
    if next_btn:
        url = base_url + next_btn['href']
    else:
        url = None
 
    print(f"これまでに{len(all_books)}冊スクレイピング...")
 
print(f"合計：{len(all_books)}冊")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
 
# ヘッドレスChromeの設定
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
 
try:
    driver.get('https://quotes.toscrape.com/js/')
 
    # コンテンツの読み込みを待機
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'quote'))
    )
 
    quotes = driver.find_elements(By.CLASS_NAME, 'quote')
    for quote in quotes:
        text = quote.find_element(By.CLASS_NAME, 'text').text
        author = quote.find_element(By.CLASS_NAME, 'author').text
        print(f'"{text}" -- {author}')
finally:
    driver.quit()

# spider.py
import scrapy
 
class BookSpider(scrapy.Spider):
    name = 'books'
    start_urls = ['https://books.toscrape.com/']
 
    def parse(self, response):
        for book in response.css('article.product_pod'):
            yield {
                'title': book.css('h3 a::attr(title)').get(),
                'price': book.css('.price_color::text').get(),
            }
 
        # ページネーションをたどる
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)
 
# 実行コマンド：scrapy runspider spider.py -o books.json

import requests
from bs4 import BeautifulSoup
import pandas as pd
 
# データをスクレイピング
url = 'https://books.toscrape.com/'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
 
books = []
for item in soup.select('article.product_pod'):
    title = item.select_one('h3 a')['title']
    price_text = item.select_one('.price_color').text
    price = float(price_text.replace('£', ''))
    rating_class = item.select_one('p.star-rating')['class'][1]
 
    books.append({
        'title': title,
        'price': price,
        'rating': rating_class,
    })
 
# DataFrameに変換
df = pd.DataFrame(books)
print(df.head())
 
# CSVに保存
df.to_csv('books.csv', index=False)

import time
import requests
 
def polite_scrape(urls, delay=2):
    """遅延付きの礼儀正しいスクレイピング。"""
    results = []
    for url in urls:
        response = requests.get(url)
        results.append(response.text)
        time.sleep(delay)  # 礼儀正しく
    return results

import pandas as pd
import pygwalker as pyg
 
df = pd.read_csv('scraped_data.csv')
walker = pyg.walk(df)