Python 웹 스크래핑: Requests, BeautifulSoup, Selenium을 사용한 완전 가이드

import requests
from bs4 import BeautifulSoup
 
# 페이지 가져오기
url = 'https://books.toscrape.com/'
response = requests.get(url)
response.raise_for_status()  # 잘못된 상태 코드에 대해 오류 발생
 
# HTML 파싱
soup = BeautifulSoup(response.text, 'html.parser')
 
# 책 제목 추출
books = soup.select('article.product_pod h3 a')
for book in books[:5]:
    print(book['title'])

# 필요한 패키지 설치
# pip install requests beautifulsoup4 lxml

from bs4 import BeautifulSoup
 
html = """
<div class="products">
    <div class="product" id="p1">
        <h2 class="name">Widget A</h2>
        <span class="price">$9.99</span>
        <p class="desc">A useful widget</p>
    </div>
    <div class="product" id="p2">
        <h2 class="name">Widget B</h2>
        <span class="price">$14.99</span>
        <p class="desc">A better widget</p>
    </div>
</div>
"""
 
soup = BeautifulSoup(html, 'html.parser')
 
# 태그로 찾기
print(soup.find('h2').text)  # "Widget A"
 
# 클래스로 찾기
prices = soup.find_all('span', class_='price')
for p in prices:
    print(p.text)  # "$9.99", "$14.99"
 
# ID로 찾기
product = soup.find(id='p2')
print(product.find('h2').text)  # "Widget B"
 
# CSS 선택자
names = soup.select('.product .name')
for n in names:
    print(n.text)

from bs4 import BeautifulSoup
 
html = '<a href="/page/2" class="next" data-page="2">Next Page</a>'
soup = BeautifulSoup(html, 'html.parser')
link = soup.find('a')
 
# 텍스트 콘텐츠
print(link.text)           # "Next Page"
print(link.get_text(strip=True))  # "Next Page" (공백 제거)
 
# 속성
print(link['href'])        # "/page/2"
print(link.get('class'))   # ['next']
print(link['data-page'])   # "2"

import requests
from bs4 import BeautifulSoup
 
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
}
 
# 세션을 사용하여 쿠키와 영구 헤더 관리
session = requests.Session()
session.headers.update(headers)
 
response = session.get('https://books.toscrape.com/')
soup = BeautifulSoup(response.text, 'html.parser')
print(f"{len(soup.select('article.product_pod'))}권의 책 발견")

import requests
from bs4 import BeautifulSoup
 
base_url = 'https://books.toscrape.com/catalogue/'
all_books = []
url = 'https://books.toscrape.com/catalogue/page-1.html'
 
while url:
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
 
    # 현재 페이지에서 책 추출
    for book in soup.select('article.product_pod'):
        title = book.select_one('h3 a')['title']
        price = book.select_one('.price_color').text
        all_books.append({'title': title, 'price': price})
 
    # 다음 페이지 찾기
    next_btn = soup.select_one('li.next a')
    if next_btn:
        url = base_url + next_btn['href']
    else:
        url = None
 
    print(f"지금까지 {len(all_books)}권 스크래핑...")
 
print(f"총: {len(all_books)}권")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
 
# 헤드리스 Chrome 설정
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
 
try:
    driver.get('https://quotes.toscrape.com/js/')
 
    # 콘텐츠 로딩 대기
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'quote'))
    )
 
    quotes = driver.find_elements(By.CLASS_NAME, 'quote')
    for quote in quotes:
        text = quote.find_element(By.CLASS_NAME, 'text').text
        author = quote.find_element(By.CLASS_NAME, 'author').text
        print(f'"{text}" -- {author}')
finally:
    driver.quit()

# spider.py
import scrapy
 
class BookSpider(scrapy.Spider):
    name = 'books'
    start_urls = ['https://books.toscrape.com/']
 
    def parse(self, response):
        for book in response.css('article.product_pod'):
            yield {
                'title': book.css('h3 a::attr(title)').get(),
                'price': book.css('.price_color::text').get(),
            }
 
        # 페이지네이션 따라가기
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)
 
# 실행 명령: scrapy runspider spider.py -o books.json

import requests
from bs4 import BeautifulSoup
import pandas as pd
 
# 데이터 스크래핑
url = 'https://books.toscrape.com/'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
 
books = []
for item in soup.select('article.product_pod'):
    title = item.select_one('h3 a')['title']
    price_text = item.select_one('.price_color').text
    price = float(price_text.replace('£', ''))
    rating_class = item.select_one('p.star-rating')['class'][1]
 
    books.append({
        'title': title,
        'price': price,
        'rating': rating_class,
    })
 
# DataFrame으로 변환
df = pd.DataFrame(books)
print(df.head())
 
# CSV로 저장
df.to_csv('books.csv', index=False)

import time
import requests
 
def polite_scrape(urls, delay=2):
    """지연을 두고 예의 바르게 스크래핑합니다."""
    results = []
    for url in urls:
        response = requests.get(url)
        results.append(response.text)
        time.sleep(delay)  # 예의 바르게
    return results

import pandas as pd
import pygwalker as pyg
 
df = pd.read_csv('scraped_data.csv')
walker = pyg.walk(df)