Python脚本爬虫Robots协议如何遵守

本文目录导读：

1. 理解Robots协议
2. Python实现遵守Robots协议
3. 集成到爬虫框架中
4. 高级处理策略
5. 最佳实践清单
6. 注意事项
7. 完整示例代码

在编写Python爬虫时，遵守robots.txt协议不仅是技术规范，更是职业道德和法律要求,以下是Python脚本爬虫遵守Robots协议的完整实践指南：

理解Robots协议

robots.txt文件位于网站根目录，https://example.com/robots.txt

主要规则：

• User-agent: 指定爬虫名称
• Disallow: 禁止访问的路径
• Allow: 允许访问的路径
• Crawl-delay: 建议的抓取间隔（秒）

Python实现遵守Robots协议

使用robotparser（Python内置库）

import urllib.robotparser
from urllib.parse import urlparse
class RobotsChecker:
    def __init__(self, base_url, user_agent='MyScraper/1.0'):
        self.base_url = base_url
        self.user_agent = user_agent
        self.rp = urllib.robotparser.RobotFileParser()
        self._load_robots()
    def _load_robots(self):
        """加载并解析robots.txt"""
        robots_url = f"{self.base_url}/robots.txt"
        self.rp.set_url(robots_url)
        try:
            self.rp.read()
        except Exception as e:
            print(f"无法读取robots.txt: {e}")
            # 保守策略：如果无法读取，默认允许所有访问
            self.rp.allow_all = True
    def can_fetch(self, url):
        """检查是否可以抓取指定URL"""
        return self.rp.can_fetch(self.user_agent, url)
    def get_crawl_delay(self):
        """获取推荐的抓取延迟"""
        delay = self.rp.crawl_delay(self.user_agent)
        return delay if delay else 1  # 默认1秒
# 使用示例
checker = RobotsChecker('https://example.com')
url = 'https://example.com/products'
if checker.can_fetch(url):
    delay = checker.get_crawl_delay()
    print(f"允许抓取 {url}，建议延迟 {delay}秒")
else:
    print(f"禁止抓取 {url}")

使用requests-robots第三方库（更便捷）

# 安装: pip install requests-robots
from requests_robots import RobotsTxt
robots = RobotsTxt('https://example.com')
# 检查URL
if robots.allowed('https://example.com/products', 'MyScraper'):
    print("允许抓取")
else:
    print("禁止抓取")
# 获取延迟
delay = robots.delay('MyScraper')
print(f"建议延迟: {delay}秒")

集成到爬虫框架中

Scrapy框架示例

import scrapy
from urllib.robotparser import RobotFileParser
class CompliantSpider(scrapy.Spider):
    name = 'compliant_spider'
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.robots_parser = RobotFileParser()
        self.robots_parser.set_url(f"{self.start_urls[0]}/robots.txt")
        self.robots_parser.read()
    def start_requests(self):
        for url in self.start_urls:
            if self.robots_parser.can_fetch(self.name, url):
                yield scrapy.Request(url, callback=self.parse)
            else:
                self.logger.warning(f"Robots协议禁止访问: {url}")
    def parse(self, response):
        # 解析逻辑
        pass

自定义通用爬虫类

import time
import requests
from urllib.robotparser import RobotFileParser
class CompliantCrawler:
    def __init__(self, base_url, user_agent='MyCrawler/1.0'):
        self.base_url = base_url.rstrip('/')
        self.user_agent = user_agent
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': user_agent})
        self.robots = RobotFileParser()
        self._init_robots()
        self.last_request_time = 0
    def _init_robots(self):
        """初始化robots解析器"""
        robots_url = f"{self.base_url}/robots.txt"
        self.robots.set_url(robots_url)
        try:
            self.robots.read()
            print(f"已加载robots.txt: {robots_url}")
        except:
            print(f"无法加载robots.txt，采用保守策略")
            self.robots.allow_all = True
    def _respect_crawl_delay(self):
        """遵守抓取延迟"""
        delay = self.robots.crawl_delay(self.user_agent)
        if delay:
            time.sleep(delay)
        else:
            time.sleep(1)  # 默认延迟
    def fetch(self, url):
        """安全抓取URL"""
        if not self.robots.can_fetch(self.user_agent, url):
            print(f"禁止访问: {url}")
            return None
        self._respect_crawl_delay()
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            # 检查响应中的meta robots标签
            if 'noindex' in response.text.lower():
                print(f"页面包含noindex指令: {url}")
            return response
        except Exception as e:
            print(f"请求失败: {e}")
            return None
# 使用示例
crawler = CompliantCrawler('https://httpbin.org')
response = crawler.fetch('https://httpbin.org/')
if response:
    print(response.status_code)

高级处理策略

处理特殊情况

class AdvancedRobotsHandler:
    def __init__(self, base_url):
        self.base_url = base_url
        self.site_maps = []  # 存储sitemap信息
        self._parse_sitemaps()
    def _parse_sitemaps(self):
        """从robots.txt中提取sitemap"""
        try:
            response = requests.get(f"{self.base_url}/robots.txt")
            for line in response.text.split('\n'):
                if line.lower().startswith('sitemap:'):
                    sitemap_url = line.split(':')[1].strip()
                    self.site_maps.append(sitemap_url)
        except:
            pass
    def get_allowed_urls(self, user_agent):
        """获取允许访问的URL列表（通过sitemap）"""
        allowed_urls = []
        for sitemap_url in self.site_maps:
            try:
                # 解析sitemap XML
                response = requests.get(sitemap_url)
                # 这里可以使用XML解析器提取URL
                # ...
            except:
                continue
        return allowed_urls
    def is_url_allowed(self, url, user_agent='*'):
        """更全面的检查"""
        # 1. 检查robots.txt
        # 2. 检查页面meta标签
        # 3. 检查X-Robots-Tag头部
        # ...
        pass

最佳实践清单

class RobotsComplianceChecklist:
    """
    遵守Robots协议的检查清单
    """
    @staticmethod
    def validate_crawler_behavior():
        checks = [
            "✅ 启动时读取并解析robots.txt",
            "✅ 使用自定义User-Agent（非默认爬虫）",
            "✅ 每次请求前检查robots.txt权限",
            "✅ 遵守Crawl-delay指令",
            "✅ 不在禁止路径下抓取",
            "✅ 尊重Disallow中的路径模式",
            "✅ 定期重新读取robots.txt（如每日一次）",
            "✅ 处理Allow例外规则",
            "✅ 检查页面meta robots标签",
            "✅ 设置合理的请求间隔",
            "✅ 不违反robots.txt中隐含的规则"
        ]
        print("Robots协议遵守检查清单:")
        for check in checks:
            print(f"  {check}")
        return checks
# 使用
RobotsComplianceChecklist.validate_crawler_behavior()

注意事项

1. 缓存策略：将robots.txt内容缓存以减少重复请求，但需设置过期时间
2. 动态网站：有些网站使用JavaScript渲染内容，但robots.txt规则仍然适用
3. 法律风险：即使遵守robots.txt，某些网站仍可能禁止爬虫
4. 道德考量：遵守robots.txt是基本的互联网礼仪

完整示例代码

#!/usr/bin/env python3
"""
遵守Robots协议的爬虫示例
"""
import time
import requests
from urllib.robotparser import RobotFileParser
from urllib.parse import urljoin
class EthicalCrawler:
    def __init__(self, base_url, user_agent='EthicalBot/1.0'):
        self.base_url = base_url.rstrip('/')
        self.user_agent = user_agent
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': user_agent})
        self.robots_parser = RobotFileParser()
        self._init_robots()
        self.last_request_time = 0
    def _init_robots(self):
        """初始化并读取robots.txt"""
        robots_url = urljoin(self.base_url, '/robots.txt')
        self.robots_parser.set_url(robots_url)
        try:
            self.robots_parser.read()
            print(f"✅ 已加载robots.txt: {robots_url}")
        except Exception as e:
            print(f"⚠️ 无法加载robots.txt: {e}")
            print("   采用保守策略：默认允许所有访问")
            self.robots_parser.allow_all = True
    def can_fetch(self, url):
        """检查是否允许抓取"""
        return self.robots_parser.can_fetch(self.user_agent, url)
    def get_delay(self):
        """获取推荐的抓取延迟"""
        delay = self.robots_parser.crawl_delay(self.user_agent)
        return max(delay, 1) if delay else 1  # 至少1秒
    def fetch(self, url, respect_robots=True):
        """安全抓取URL"""
        full_url = urljoin(self.base_url, url)
        # 步骤1: 检查robots.txt权限
        if respect_robots and not self.can_fetch(full_url):
            print(f"❌ 禁止访问: {full_url}")
            return None
        # 步骤2: 遵守Crawl-Delay
        delay = self.get_delay()
        time_since_last = time.time() - self.last_request_time
        if time_since_last < delay:
            wait_time = delay - time_since_last
            print(f"⏳ 等待 {wait_time:.2f} 秒...")
            time.sleep(wait_time)
        # 步骤3: 发送请求
        try:
            print(f"➡️ 正在抓取: {full_url}")
            self.last_request_time = time.time()
            response = self.session.get(full_url, timeout=10)
            response.raise_for_status()
            # 步骤4: 检查页面级meta robots标签
            if 'noindex' in response.text.lower()[:5000]:
                print(f"⚠️ 页面包含noindex指令: {full_url}")
                # 可根据策略决定是否处理
            print(f"✅ 成功抓取: {response.status_code}")
            return response
        except requests.exceptions.RequestException as e:
            print(f"❌ 请求失败: {e}")
            return None
# 使用示例
if __name__ == "__main__":
    crawler = EthicalCrawler('https://httpbin.org')
    # 抓取允许的页面
    response = crawler.fetch('/')
    if response:
        print(response.text[:200])
    # 抓取禁止的页面（会失败）
    crawler.fetch('/deny-page')

通过以上方法，你的Python爬虫可以正确遵守Robots协议，既尊重网站规则，又能高效合法地获取数据,记得始终将遵守协议放在首位！