IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

PHP批量抓取网页标题

wen PHP项目 1

本文目录导读：

PHP批量抓取网页标题

  1. 方案一：简单版（使用file_get_contents）
  2. 方案二：使用cURL（推荐，更稳定）
  3. 方案三：带数据库存储的完整版
  4. 使用建议

我来帮你写一个PHP批量抓取网页标题的脚本,这里有几种方案，从简单到复杂：

简单版（使用file_get_contents）

<?php
/**
 * 简单版：批量抓取网页标题
 * 使用 file_get_contents 直接获取网页内容
 */
// 要抓取的URL列表
$urls = [
    'https://www.baidu.com',
    'https://www.qq.com',
    'https://www.163.com',
    'https://www.sina.com.cn'
];
foreach ($urls as $url) {
    try {
        // 设置超时和上下文
        $context = stream_context_create([
            'http' => [
                'timeout' => 10,
                'header' => "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36\r\n"
            ]
        ]);
        // 获取网页内容
        $html = @file_get_contents($url, false, $context);
        if ($html !== false) {
            // 匹配标题
            preg_match('/<title[^>]*>(.*?)<\/title>/i', $html, $matches);
            $title = isset($matches[1]) ? trim($matches[1]) : '未找到标题';
            echo "URL: $url\n";
            echo "标题: $title\n";
            echo "-------------------\n";
        } else {
            echo "URL: $url\n";
            echo "标题: 获取失败\n";
            echo "-------------------\n";
        }
    } catch (Exception $e) {
        echo "URL: $url\n";
        echo "错误: " . $e->getMessage() . "\n";
        echo "-------------------\n";
    }
}
?>

使用cURL（推荐，更稳定）

<?php
/**
 * 使用cURL批量抓取网页标题
 * cURL更稳定，支持并发请求
 */
class BatchTitleFetcher {
    private $urls = [];
    private $timeout = 10;
    private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
    public function __construct(array $urls, $timeout = 10) {
        $this->urls = $urls;
        $this->timeout = $timeout;
    }
    /**
     * 单个抓取
     */
    public function fetchSingle($url) {
        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_USERAGENT => $this->userAgent,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,
            CURLOPT_ENCODING => '',
        ]);
        $html = curl_exec($ch);
        $error = curl_error($ch);
        curl_close($ch);
        if ($html === false) {
            return ['error' => $error];
        }
        // 提取标题
        if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $html, $matches)) {
            return ['title' => trim($matches[1])];
        }
        return ['error' => '未找到标题'];
    }
    /**
     * 批量抓取（串行）
     */
    public function fetchAllSerial() {
        $results = [];
        foreach ($this->urls as $url) {
            echo "正在抓取: $url\n";
            $results[$url] = $this->fetchSingle($url);
            sleep(1); // 防止请求过快
        }
        return $results;
    }
    /**
     * 批量抓取（并发）
     */
    public function fetchAllParallel() {
        $results = [];
        $handles = [];
        $mh = curl_multi_init();
        // 创建多个cURL句柄
        foreach ($this->urls as $key => $url) {
            $ch = curl_init();
            curl_setopt_array($ch, [
                CURLOPT_URL => $url,
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_TIMEOUT => $this->timeout,
                CURLOPT_USERAGENT => $this->userAgent,
                CURLOPT_SSL_VERIFYPEER => false,
                CURLOPT_SSL_VERIFYHOST => false,
                CURLOPT_ENCODING => '',
            ]);
            curl_multi_add_handle($mh, $ch);
            $handles[$key] = $ch;
            $results[$url] = ['status' => 'pending'];
        }
        // 执行并发请求
        $running = null;
        do {
            curl_multi_exec($mh, $running);
            curl_multi_select($mh);
        } while ($running > 0);
        // 获取结果
        foreach ($handles as $key => $ch) {
            $url = $this->urls[$key];
            $html = curl_multi_getcontent($ch);
            $error = curl_error($ch);
            if ($html !== false && empty($error)) {
                if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $html, $matches)) {
                    $results[$url] = ['title' => trim($matches[1])];
                } else {
                    $results[$url] = ['error' => '未找到标题'];
                }
            } else {
                $results[$url] = ['error' => $error ?: '获取失败'];
            }
            curl_multi_remove_handle($mh, $ch);
            curl_close($ch);
        }
        curl_multi_close($mh);
        return $results;
    }
    /**
     * 格式化输出结果
     */
    public function printResults(array $results) {
        echo "\n========== 抓取结果 ==========\n\n";
        foreach ($results as $url => $result) {
            echo "URL: $url\n";
            if (isset($result['title'])) {
                echo "标题: {$result['title']}\n";
            } elseif (isset($result['error'])) {
                echo "错误: {$result['error']}\n";
            } else {
                echo "状态: {$result['status']}\n";
            }
            echo "-------------------\n";
        }
    }
}
// 使用示例
$urls = [
    'https://www.baidu.com',
    'https://www.qq.com',
    'https://www.163.com',
    'https://www.sina.com.cn',
    'https://www.taobao.com'
];
$fetcher = new BatchTitleFetcher($urls, 10);
echo "=== 串行抓取 ===\n";
$results = $fetcher->fetchAllSerial();
$fetcher->printResults($results);
echo "\n\n=== 并发抓取 ===\n";
$results = $fetcher->fetchAllParallel();
$fetcher->printResults($results);
?>

带数据库存储的完整版

<?php
/**
 * 完整版：批量抓取并保存到数据库
 */
class TitleFetcher {
    private $pdo;
    private $timeout = 10;
    public function __construct($dbConfig) {
        // 数据库连接
        $dsn = "mysql:host={$dbConfig['host']};dbname={$dbConfig['dbname']};charset=utf8mb4";
        $this->pdo = new PDO($dsn, $dbConfig['user'], $dbConfig['password']);
        $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
        $this->createTable();
    }
    private function createTable() {
        $sql = "CREATE TABLE IF NOT EXISTS web_titles (
            id INT AUTO_INCREMENT PRIMARY KEY,
            url VARCHAR(500) NOT NULL,
            title VARCHAR(500),
            status VARCHAR(20) DEFAULT 'success',
            error_message TEXT,
            fetch_time DATETIME DEFAULT CURRENT_TIMESTAMP,
            UNIQUE KEY unique_url (url(191))
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
        $this->pdo->exec($sql);
    }
    public function fetchAndSave($url) {
        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,
        ]);
        $html = curl_exec($ch);
        $error = curl_error($ch);
        curl_close($ch);
        if ($html === false) {
            $this->saveResult($url, null, 'error', $error);
            return false;
        }
        // 提取标题
        if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $html, $matches)) {
            $title = trim($matches[1]);
            $this->saveResult($url, $title, 'success', null);
            return $title;
        }
        $this->saveResult($url, null, 'no_title', '未找到标题');
        return null;
    }
    private function saveResult($url, $title, $status, $error) {
        $sql = "INSERT INTO web_titles (url, title, status, error_message, fetch_time) 
                VALUES (:url, :title, :status, :error, NOW())
                ON DUPLICATE KEY UPDATE 
                title = VALUES(title),
                status = VALUES(status),
                error_message = VALUES(error_message),
                fetch_time = NOW()";
        $stmt = $this->pdo->prepare($sql);
        $stmt->execute([
            ':url' => $url,
            ':title' => $title,
            ':status' => $status,
            ':error' => $error
        ]);
    }
    public function batchFetch($urls) {
        $results = [];
        foreach ($urls as $index => $url) {
            echo "[" . ($index + 1) . "/" . count($urls) . "] 正在处理: $url\n";
            $title = $this->fetchAndSave($url);
            $results[$url] = $title;
            if ($index < count($urls) - 1) {
                // 延迟1-2秒，避免被封IP
                sleep(rand(1, 2));
            }
        }
        return $results;
    }
    public function getHistory($limit = 50) {
        $sql = "SELECT * FROM web_titles ORDER BY fetch_time DESC LIMIT $limit";
        return $this->pdo->query($sql)->fetchAll(PDO::FETCH_ASSOC);
    }
}
// 使用示例
$dbConfig = [
    'host' => 'localhost',
    'dbname' => 'your_database',
    'user' => 'root',
    'password' => 'your_password'
];
$fetcher = new TitleFetcher($dbConfig);
$urls = [
    'https://www.baidu.com',
    'https://www.qq.com',
    'https://www.163.com',
    'https://www.sina.com.cn',
    'https://www.taobao.com',
    'https://www.jd.com',
    'https://www.tmall.com'
];
// 批量抓取
$results = $fetcher->batchFetch($urls);
// 显示结果
echo "\n========== 抓取结果 ==========\n\n";
foreach ($results as $url => $title) {
    echo "URL: $url\n";
    echo "标题: " . ($title ?? "失败") . "\n";
    echo "-------------------\n";
}
// 查看历史记录
echo "\n========== 历史记录 ==========\n\n";
$history = $fetcher->getHistory(10);
foreach ($history as $row) {
    echo "URL: {$row['url']}\n";
    echo "标题: {$row['title']}\n";
    echo "状态: {$row['status']}\n";
    echo "时间: {$row['fetch_time']}\n";
    echo "-------------------\n";
}
?>

使用建议

  1. 选择合适的方案：

    • 简单抓取用file_get_contents
    • 需要稳定性用cURL
    • 需要持久化用数据库版本

  2. 性能优化：

    • 使用cURL多线程并发请求
    • 设置合理的超时时间
    • 控制请求频率,避免被封

  3. 注意事项：

    • 遵守目标网站robots.txt
    • 添加适当的User-Agent
    • 处理SSL证书问题
    • 支持字符编码转换

如果还有问题,请告诉我具体需求，我可以帮你调整代码！

上一篇PHP多维数组排序技巧

下一篇PHP简单词频统计脚本

相关文章

抱歉，评论功能暂时关闭!