scraper/
├── scraper/
│   ├── __init__.py
│   ├── core.py         # 爬虫核心
│   ├── parser.py       # 页面解析
│   ├── storage.py      # 数据存储
│   └── utils.py        # 工具函数
├── config.py           # 配置
└── main.py             # 入口

# scraper/core.py
import asyncio
import aiohttp
from typing import List, Dict
from .utils import RateLimiter, RetryHandler

class AsyncScraper:
    def __init__(self, max_concurrent: int = 5, delay: float = 1.0):
        self.max_concurrent = max_concurrent
        self.rate_limiter = RateLimiter(delay)
        self.retry_handler = RetryHandler(max_retries=3)
        self.session = None

    async def __aenter__(self):
        self.session = aiohttp.ClientSession()
        return self

    async def __aexit__(self, *args):
        if self.session:
            await self.session.close()

    async def fetch(self, url: str) -> str:
        """获取单个页面"""
        await self.rate_limiter.acquire()

        async def _fetch():
            async with self.session.get(url) as response:
                response.raise_for_status()
                return await response.text()

        return await self.retry_handler.execute(_fetch)

    async def fetch_many(self, urls: List[str]) -> Dict[str, str]:
        """并发获取多个页面"""
        semaphore = asyncio.Semaphore(self.max_concurrent)

        async def _fetch_with_semaphore(url):
            async with semaphore:
                try:
                    content = await self.fetch(url)
                    return url, content
                except Exception as e:
                    print(f"Failed to fetch {url}: {e}")
                    return url, None

        tasks = [_fetch_with_semaphore(url) for url in urls]
        results = await asyncio.gather(*tasks)
        return {url: content for url, content in results if content}

# scraper/utils.py
import asyncio
import time
from functools import wraps

class RateLimiter:
    """速率限制器"""
    def __init__(self, delay: float):
        self.delay = delay
        self.last_request = 0

    async def acquire(self):
        now = time.time()
        elapsed = now - self.last_request
        if elapsed < self.delay:
            await asyncio.sleep(self.delay - elapsed)
        self.last_request = time.time()

class RetryHandler:
    """重试处理器"""
    def __init__(self, max_retries: int = 3, backoff: float = 1.0):
        self.max_retries = max_retries
        self.backoff = backoff

    async def execute(self, func, *args, **kwargs):
        for attempt in range(self.max_retries):
            try:
                return await func(*args, **kwargs)
            except Exception as e:
                if attempt == self.max_retries - 1:
                    raise
                wait = self.backoff * (2 ** attempt)
                print(f"Retry {attempt + 1}/{self.max_retries} after {wait}s: {e}")
                await asyncio.sleep(wait)

# scraper/parser.py
from bs4 import BeautifulSoup
from typing import List, Dict

def parse_article_list(html: str) -> List[Dict]:
    """解析文章列表页"""
    soup = BeautifulSoup(html, "html.parser")
    articles = []

    for item in soup.select("article.post"):
        title = item.select_one("h2 a")
        summary = item.select_one(".summary")
        date = item.select_one("time")

        articles.append({
            "title": title.text.strip() if title else "",
            "url": title["href"] if title else "",
            "summary": summary.text.strip() if summary else "",
            "date": date["datetime"] if date else "",
        })

    return articles

def parse_article_detail(html: str) -> Dict:
    """解析文章详情页"""
    soup = BeautifulSoup(html, "html.parser")

    content = soup.select_one("article.content")
    author = soup.select_one(".author-name")
    tags = soup.select("span.tag")

    return {
        "content": content.text.strip() if content else "",
        "author": author.text.strip() if author else "",
        "tags": [t.text.strip() for t in tags],
    }

# scraper/storage.py
import csv
import json
from pathlib import Path
from typing import List, Dict

class DataStorage:
    def __init__(self, output_dir: str = "output"):
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(exist_ok=True)

    def save_to_csv(self, data: List[Dict], filename: str):
        """保存为 CSV"""
        if not data:
            return

        filepath = self.output_dir / filename
        with open(filepath, "w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=data[0].keys())
            writer.writeheader()
            writer.writerows(data)

        print(f"Saved {len(data)} records to {filepath}")

    def save_to_json(self, data: List[Dict], filename: str):
        """保存为 JSON"""
        filepath = self.output_dir / filename
        with open(filepath, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)

        print(f"Saved {len(data)} records to {filepath}")

# main.py
import asyncio
from scraper.core import AsyncScraper
from scraper.parser import parse_article_list, parse_article_detail
from scraper.storage import DataStorage

async def main():
    storage = DataStorage()

    async with AsyncScraper(max_concurrent=3, delay=1.5) as scraper:
        # 1. 获取文章列表
        list_urls = [f"https://example.com/page/{i}" for i in range(1, 6)]
        list_pages = await scraper.fetch_many(list_urls)

        # 2. 解析文章列表
        all_articles = []
        for url, html in list_pages.items():
            articles = parse_article_list(html)
            all_articles.extend(articles)

        print(f"Found {len(all_articles)} articles")

        # 3. 获取文章详情
        detail_urls = [a["url"] for a in all_articles if a["url"]]
        detail_pages = await scraper.fetch_many(detail_urls[:10])  # 限制数量

        # 4. 合并数据
        for article in all_articles:
            if article["url"] in detail_pages:
                detail = parse_article_detail(detail_pages[article["url"]])
                article.update(detail)

        # 5. 保存数据
        storage.save_to_csv(all_articles, "articles.csv")
        storage.save_to_json(all_articles, "articles.json")

if __name__ == "__main__":
    asyncio.run(main())

scraper/
├── scraper/
│   ├── __init__.py
│   ├── core.py         # 异步爬虫核心（限流/并发/重试）
│   ├── parser.py       # 页面解析（列表/详情）
│   ├── storage.py      # 数据存储（CSV/JSON）
│   └── utils.py        # 工具函数（限速器/重试器）
├── config.py           # 配置管理
└── main.py             # 主程序