Web Scraping mit Python: Vollständige Anleitung mit Requests, BeautifulSoup und Selenium

import requests
from bs4 import BeautifulSoup
 
# Seite abrufen
url = 'https://books.toscrape.com/'
response = requests.get(url)
response.raise_for_status()  # Fehler bei schlechtem Statuscode auslösen
 
# HTML parsen
soup = BeautifulSoup(response.text, 'html.parser')
 
# Buchtitel extrahieren
books = soup.select('article.product_pod h3 a')
for book in books[:5]:
    print(book['title'])

# Erforderliche Pakete installieren
# pip install requests beautifulsoup4 lxml

from bs4 import BeautifulSoup
 
html = """
<div class="products">
    <div class="product" id="p1">
        <h2 class="name">Widget A</h2>
        <span class="price">$9.99</span>
        <p class="desc">A useful widget</p>
    </div>
    <div class="product" id="p2">
        <h2 class="name">Widget B</h2>
        <span class="price">$14.99</span>
        <p class="desc">A better widget</p>
    </div>
</div>
"""
 
soup = BeautifulSoup(html, 'html.parser')
 
# Nach Tag suchen
print(soup.find('h2').text)  # "Widget A"
 
# Nach Klasse suchen
prices = soup.find_all('span', class_='price')
for p in prices:
    print(p.text)  # "$9.99", "$14.99"
 
# Nach ID suchen
product = soup.find(id='p2')
print(product.find('h2').text)  # "Widget B"
 
# CSS-Selektor
names = soup.select('.product .name')
for n in names:
    print(n.text)

from bs4 import BeautifulSoup
 
html = '<a href="/page/2" class="next" data-page="2">Next Page</a>'
soup = BeautifulSoup(html, 'html.parser')
link = soup.find('a')
 
# Textinhalt
print(link.text)           # "Next Page"
print(link.get_text(strip=True))  # "Next Page" (bereinigt)
 
# Attribute
print(link['href'])        # "/page/2"
print(link.get('class'))   # ['next']
print(link['data-page'])   # "2"

import requests
from bs4 import BeautifulSoup
 
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
}
 
# Session für Cookies und persistente Header verwenden
session = requests.Session()
session.headers.update(headers)
 
response = session.get('https://books.toscrape.com/')
soup = BeautifulSoup(response.text, 'html.parser')
print(f"{len(soup.select('article.product_pod'))} Bücher gefunden")

import requests
from bs4 import BeautifulSoup
 
base_url = 'https://books.toscrape.com/catalogue/'
all_books = []
url = 'https://books.toscrape.com/catalogue/page-1.html'
 
while url:
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
 
    # Bücher von der aktuellen Seite extrahieren
    for book in soup.select('article.product_pod'):
        title = book.select_one('h3 a')['title']
        price = book.select_one('.price_color').text
        all_books.append({'title': title, 'price': price})
 
    # Nächste Seite finden
    next_btn = soup.select_one('li.next a')
    if next_btn:
        url = base_url + next_btn['href']
    else:
        url = None
 
    print(f"Bisher {len(all_books)} Bücher gescrapt...")
 
print(f"Gesamt: {len(all_books)} Bücher")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
 
# Headless Chrome einrichten
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
 
try:
    driver.get('https://quotes.toscrape.com/js/')
 
    # Auf Inhaltsladung warten
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'quote'))
    )
 
    quotes = driver.find_elements(By.CLASS_NAME, 'quote')
    for quote in quotes:
        text = quote.find_element(By.CLASS_NAME, 'text').text
        author = quote.find_element(By.CLASS_NAME, 'author').text
        print(f'"{text}" -- {author}')
finally:
    driver.quit()

# spider.py
import scrapy
 
class BookSpider(scrapy.Spider):
    name = 'books'
    start_urls = ['https://books.toscrape.com/']
 
    def parse(self, response):
        for book in response.css('article.product_pod'):
            yield {
                'title': book.css('h3 a::attr(title)').get(),
                'price': book.css('.price_color::text').get(),
            }
 
        # Paginierung folgen
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)
 
# Ausführen mit: scrapy runspider spider.py -o books.json

import requests
from bs4 import BeautifulSoup
import pandas as pd
 
# Daten scrapen
url = 'https://books.toscrape.com/'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
 
books = []
for item in soup.select('article.product_pod'):
    title = item.select_one('h3 a')['title']
    price_text = item.select_one('.price_color').text
    price = float(price_text.replace('£', ''))
    rating_class = item.select_one('p.star-rating')['class'][1]
 
    books.append({
        'title': title,
        'price': price,
        'rating': rating_class,
    })
 
# In DataFrame konvertieren
df = pd.DataFrame(books)
print(df.head())
 
# Als CSV speichern
df.to_csv('books.csv', index=False)

import time
import requests
 
def polite_scrape(urls, delay=2):
    """Scraping mit Verzögerungen zwischen Anfragen."""
    results = []
    for url in urls:
        response = requests.get(url)
        results.append(response.text)
        time.sleep(delay)  # Höflich bleiben
    return results

import pandas as pd
import pygwalker as pyg
 
df = pd.read_csv('scraped_data.csv')
walker = pyg.walk(df)