Webスクレイピング：requests×BeautifulSoupの基本｜“合法×丁寧×再現性”でデータ取得を設計する

requests>=2.32
beautifulsoup4>=4.12
lxml>=5
pandas>=2.2

project/
  scrape/
    __init__.py
    main.py
    fetch.py
    parse.py
    io_utils.py
  tests/
    test_parse.py
  data/
    raw/
    out/

from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser

def is\_allowed(url: str, user\_agent: str = "pythonbunseki-bot/1.0") -> bool:
u = urlparse(url)
robots = f"{u.scheme}://{u.netloc}/robots.txt"
rp = RobotFileParser()
try:
rp.set\_url(robots)
rp.read()
return rp.can\_fetch(user\_agent, url)
except Exception:
\# 取得不可は安全側でFalse（人の判断を挟む）
return False

import time, random, requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

UA = "pythonbunseki-bot/1.0 (+[https://pythonbunseki.com](https://pythonbunseki.com))"

def make\_session() -> requests.Session:
s = requests.Session()
s.headers.update({
"User-Agent": UA,
"Accept-Language": "ja,en;q=0.8",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
})
retry = Retry(
total=3,
backoff\_factor=0.7,
status\_forcelist=\[429, 500, 502, 503, 504],
allowed\_methods=\["GET", "HEAD"],
respect\_retry\_after\_header=True,
)
s.mount("http\://", HTTPAdapter(max\_retries=retry))
s.mount("https\://", HTTPAdapter(max\_retries=retry))
return s

def get\_html(s: requests.Session, url: str, \*, min\_delay=1.0, max\_delay=2.5, timeout=10) -> str:
if not is\_allowed(url, UA):
raise PermissionError(f"robots.txt disallow: {url}")
r = s.get(url, timeout=timeout)
if r.status\_code == 429:
raise RuntimeError("rate limited: 429")  # ここで停止/再開判断
r.raise\_for\_status()
if not r.encoding:
r.encoding = r.apparent\_encoding
html = r.text
time.sleep(random.uniform(min\_delay, max\_delay))  # 人間より遅く
return html

from bs4 import BeautifulSoup
from dataclasses import dataclass

@dataclass
class Item:
title: str
price: int | None
url: str

def parse\_list(html: str) -> list\[str]:
soup = BeautifulSoup(html, "lxml")
links = \[a\["href"] for a in soup.select("a.item-link\[href]")]
return links

def parse\_detail(html: str, url: str) -> Item:
soup = BeautifulSoup(html, "lxml")
title = soup.select\_one("h1.item-title").get\_text(strip=True)
price\_el = soup.select\_one(".price .amount")
price = None
if price\_el:
txt = price\_el.get\_text(strip=True).replace(",", "")
price = int(txt) if txt.isdigit() else None
return Item(title=title, price=price, url=url)

from bs4 import BeautifulSoup

def table\_to\_dicts(html: str, selector: str) -> list\[dict]:
soup = BeautifulSoup(html, "lxml")
table = soup.select\_one(selector)
headers = \[th.get\_text(strip=True) for th in table.select("thead th")]
rows = \[]
for tr in table.select("tbody tr"):
cells = \[td.get\_text(strip=True) for td in tr.select("td")]
rows.append(dict(zip(headers, cells)))
return rows

from urllib.parse import urljoin
import pandas as pd
from scrape.fetch import make_session, get_html
from scrape.parse import parse_list, parse_detail

BASE = "[https://example.com/](https://example.com/)"

def iter\_pages() -> list\[str]:
return \[f"{BASE}list?page={i}" for i in range(1, 4)]

def run() -> pd.DataFrame:
s = make\_session()
items = \[]
for lp in iter\_pages():
html = get\_html(s, lp)
rels = parse\_list(html)
for rel in rels:
url = urljoin(BASE, rel)
detail\_html = get\_html(s, url)
item = parse\_detail(detail\_html, url)
items.append(item.**dict**)
df = pd.DataFrame(items)
return df

if **name** == "**main**":
df = run()
df.to\_csv("data/out/items.csv", index=False, encoding="utf-8")
df.to\_parquet("data/out/items.parquet")

from pathlib import Path
import tempfile, os

def atomic\_write\_text(path: Path, text: str, encoding: str = "utf-8") -> None:
path.parent.mkdir(parents=True, exist\_ok=True)
with tempfile.NamedTemporaryFile("w", delete=False, dir=path.parent, encoding=encoding, newline="") as tmp:
tmp.write(text)
tmp\_path = Path(tmp.name)
os.replace(tmp\_path, path)

from scrape.parse import parse_list, parse_detail

LIST\_HTML = """
  <a class="item-link" href="/d/1">A</a>
  <a class="item-link" href="/d/2">B</a>

"""
DETAIL_HTML = """
  <h1 class="item-title">サンプル商品</h1>
  <div class="price"><span class="amount">1,280</span>円</div>

"""

def test\_parse\_list():
assert parse\_list(LIST\_HTML) == \["/d/1", "/d/2"]

def test\_parse\_detail():
it = parse\_detail(DETAIL\_HTML, "[https://example.com/d/1](https://example.com/d/1)")
assert it.title == "サンプル商品" and it.price == 1280