PHP简易爬虫框架怎么搭

wen PHP项目 2026-08-11 232

本文目录导读：

PHP简易爬虫框架怎么搭

项目结构
核心类实现
解析器接口和实现
存储接口和实现
配置文件
使用示例
Composer 配置文件
高级功能
测试文件
运行命令
扩展建议

我来帮你搭建一个PHP简易爬虫框架,我们需要设计一个灵活且可扩展的架构。

项目结构

crawler/
├── src/
│   ├── Core/
│   │   ├── Crawler.php
│   │   ├── Request.php
│   │   └── Response.php
│   ├── Parser/
│   │   ├── ParserInterface.php
│   │   └── HTMLParser.php
│   ├── Storage/
│   │   ├── StorageInterface.php
│   │   └── DatabaseStorage.php
│   └── Utils/
│       └── Helper.php
├── config/
│   └── config.php
├── tests/
└── composer.json

核心类实现

Request.php

<?php
namespace Crawler\Core;
class Request {
    private $url;
    private $method;
    private $headers = [];
    private $body = [];
    private $options = [];
    private $timeout = 30;
    private $maxRetries = 3;
    private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
    private $proxy = null;
    public function __construct($url, $method = 'GET', $headers = [], $body = []) {
        $this->url = $url;
        $this->method = $method;
        $this->headers = $headers;
        $this->body = $body;
    }
    public function getUrl() { return $this->url; }
    public function setUrl($url) { $this->url = $url; return $this; }
    public function getMethod() { return $this->method; }
    public function setMethod($method) { $this->method = $method; return $this; }
    public function addHeader($key, $value) {
        $this->headers[$key] = $value;
        return $this;
    }
    public function getHeaders() { return $this->headers; }
    public function setBody($data) { 
        $this->body = $data; 
        return $this;
    }
    public function getBody() { return $this->body; }
    public function setTimeout($timeout) { $this->timeout = $timeout; return $this; }
    public function getTimeout() { return $this->timeout; }
    public function setUserAgent($ua) { $this->userAgent = $ua; return $this; }
    public function getUserAgent() { return $this->userAgent; }
    public function setProxy($proxy) { $this->proxy = $proxy; return $this; }
    public function getProxy() { return $this->proxy; }
    public function toCurlOptions() {
        $curlOptions = [
            CURLOPT_URL => $this->url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_USERAGENT => $this->userAgent,
            CURLOPT_HEADER => false
        ];
        // 设置请求头
        if (!empty($this->headers)) {
            $curlOptions[CURLOPT_HTTPHEADER] = $this->headers;
        }
        // 设置请求方法
        switch (strtoupper($this->method)) {
            case 'POST':
                $curlOptions[CURLOPT_POST] = true;
                $curlOptions[CURLOPT_POSTFIELDS] = $this->body;
                break;
            case 'PUT':
                $curlOptions[CURLOPT_CUSTOMREQUEST] = 'PUT';
                $curlOptions[CURLOPT_POSTFIELDS] = $this->body;
                break;
            case 'DELETE':
                $curlOptions[CURLOPT_CUSTOMREQUEST] = 'DELETE';
                break;
            case 'HEAD':
                $curlOptions[CURLOPT_NOBODY] = true;
                break;
        }
        // 设置代理
        if ($this->proxy) {
            $curlOptions[CURLOPT_PROXY] = $this->proxy;
        }
        return $curlOptions;
    }
}

Response.php

<?php
namespace Crawler\Core;
class Response {
    private $statusCode;
    private $headers = [];
    private $body;
    private $error;
    private $executionTime;
    private $finalUrl;
    public function __construct($statusCode, $headers, $body, $error = null, $executionTime = 0, $finalUrl = null) {
        $this->statusCode = $statusCode;
        $this->headers = $headers;
        $this->body = $body;
        $this->error = $error;
        $this->executionTime = $executionTime;
        $this->finalUrl = $finalUrl;
    }
    public function getStatusCode() { return $this->statusCode; }
    public function getHeaders() { return $this->headers; }
    public function getBody() { return $this->body; }
    public function getError() { return $this->error; }
    public function getExecutionTime() { return $this->executionTime; }
    public function getFinalUrl() { return $this->finalUrl; }
    public function isSuccess() {
        return $this->statusCode >= 200 && $this->statusCode < 300;
    }
    public function toArray() {
        return [
            'status_code' => $this->statusCode,
            'headers' => $this->headers,
            'body' => $this->body,
            'error' => $this->error,
            'execution_time' => $this->executionTime,
            'final_url' => $this->finalUrl
        ];
    }
}

Crawler.php

<?php
namespace Crawler\Core;
use Crawler\Parser\ParserInterface;
use Crawler\Storage\StorageInterface;
class Crawler {
    protected $request;
    protected $response;
    protected $parser;
    protected $storage;
    protected $config = [];
    // 爬取统计
    protected $stats = [
        'total_requests' => 0,
        'total_pages' => 0,
        'total_bytes' => 0,
        'errors' => 0,
        'start_time' => null,
        'end_time' => null
    ];
    // 回调函数
    protected $callbacks = [
        'onSuccess' => null,
        'onError' => null,
        'onParse' => null
    ];
    public function __construct(array $config = []) {
        $this->config = array_merge([
            'max_retries' => 3,
            'timeout' => 30,
            'concurrency' => 1,
            'use_random_user_agent' => false,
            'respect_robots_txt' => true,
            'user_agents' => $this->getDefaultUserAgents()
        ], $config);
        $this->init();
    }
    protected function init() {
        // 可以在这里初始化数据库连接等
    }
    public function setRequest(Request $request) {
        $this->request = $request;
        return $this;
    }
    public function setParser(ParserInterface $parser) {
        $this->parser = $parser;
        return $this;
    }
    public function setStorage(StorageInterface $storage) {
        $this->storage = $storage;
        return $this;
    }
    public function setCallback($event, $callback) {
        $this->callbacks[$event] = $callback;
        return $this;
    }
    public function fetch($url) {
        $this->stats['start_time'] = microtime(true);
        try {
            $this->makeRequest($url);
            if ($this->response->isSuccess()) {
                $this->stats['total_pages']++;
                $this->handleSuccess();
            } else {
                $this->stats['errors']++;
                $this->handleError();
            }
            return $this->response;
        } finally {
            $this->stats['end_time'] = microtime(true);
        }
    }
    protected function makeRequest($url) {
        $startTime = microtime(true);
        $retries = 0;
        do {
            $ch = curl_init();
            $request = new Request($url);
            if (!empty($this->request->getHeaders())) {
                foreach ($this->request->getHeaders() as $key => $value) {
                    $request->addHeader($key, $value);
                }
            }
            // 随机User-Agent
            if ($this->config['use_random_user_agent']) {
                $randomUA = $this->config['user_agents'][array_rand($this->config['user_agents'])];
                $request->setUserAgent($randomUA);
            } else {
                $request->setUserAgent($this->request->getUserAgent());
            }
            $options = $request->toCurlOptions();
            curl_setopt_array($ch, $options);
            // 发送请求
            $body = curl_exec($ch);
            $error = curl_error($ch);
            $info = curl_getinfo($ch);
            curl_close($ch);
            if ($error) {
                $this->showMessage("Request error: " . $error, 'error');
                $retries++;
                if ($retries >= $this->config['max_retries']) {
                    break;
                }
                sleep(1); // 等待1秒后重试
                continue;
            }
            break;
        } while (true);
        $executionTime = microtime(true) - $startTime;
        $this->response = new Response(
            isset($info['http_code']) ? $info['http_code'] : 0,
            $this->parseHeaders($info),
            $body,
            $error,
            $executionTime,
            isset($info['url']) ? $info['url'] : $url
        );
        $this->stats['total_requests']++;
        $this->stats['total_bytes'] += strlen($body);
    }
    protected function handleSuccess() {
        // 输出响应信息
        $this->showMessage("获取成功: " . $this->response->getFinalUrl(), 'success');
        // 执行解析
        if ($this->parser !== null) {
            $data = $this->parser->parse($this->response->getBody());
            if ($this->callbacks['onParse']) {
                $data = call_user_func($this->callbacks['onParse'], $data, $this->response);
            }
            // 存储数据
            if ($this->storage !== null && $data) {
                $this->storage->save($data);
            }
        }
        // 执行成功回调
        if ($this->callbacks['onSuccess']) {
            call_user_func($this->callbacks['onSuccess'], $this->response);
        }
    }
    protected function handleError() {
        $this->showMessage("请求失败，状态码: " . $this->response->getStatusCode(), 'error');
        if ($this->callbacks['onError']) {
            call_user_func($this->callbacks['onError'], $this->response);
        }
    }
    protected function parseHeaders($info) {
        // 这个方法需要从curl响应中提取headers
        // 简化实现，实际可以从$info获取
        return [];
    }
    public function getStats() {
        $this->stats['duration'] = $this->stats['end_time'] - $this->stats['start_time'];
        return $this->stats;
    }
    protected function showMessage($message, $type = 'info') {
        if (PHP_SAPI === 'cli') {
            $colors = [
                'success' => "\033[32m",
                'error' => "\033[31m",
                'info' => "\033[33m",
                'reset' => "\033[0m"
            ];
            echo $colors[$type] . $message . $colors['reset'] . PHP_EOL;
        } else {
            echo $message . "<br>";
        }
    }
    protected function getDefaultUserAgents() {
        return [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15'
        ];
    }
}

解析器接口和实现

ParserInterface.php

<?php
namespace Crawler\Parser;
interface ParserInterface {
    public function parse($html);
}

HTMLParser.php

<?php
namespace Crawler\Parser;
use DOMDocument;
use DOMXPath;
class HTMLParser implements ParserInterface {
    protected $xpath = null;
    protected $config = [];
    public function __construct($config = []) {
        $this->config = array_merge([
            'selectors' => [],
            'remove_tags' => [],
            'encoding' => 'UTF-8'
        ], $config);
    }
    public function parse($html) {
        $document = new DOMDocument();
        libxml_use_internal_errors(true);
        @$document->loadHTML('<?xml encoding="' . $this->config['encoding'] . '">' . $html);
        libxml_clear_errors();
        $this->xpath = new DOMXPath($document);
        $result = [];
        foreach ($this->config['selectors'] as $name => $selector) {
            $result[$name] = $this->extractBySelector($selector);
        }
        return $result;
    }
    protected function extractBySelector($selector) {
        // 支持简单的CSS选择器转XPath
        $xpath = $this->cssToXPath($selector['selector']);
        if (isset($selector['attribute'])) {
            return $this->getAttribute($xpath, $selector['attribute']);
        } else {
            return $this->getText($xpath);
        }
    }
    protected function cssToXPath($css) {
        // 简单实现，可根据需要扩展
        // 这里只处理最基本的后代选择器
        $parts = explode(' ', trim($css));
        $xpath = '';
        foreach ($parts as $part) {
            $xpath .= '/' . $part;
        }
        return $xpath;
    }
    protected function getText($xpath) {
        $nodes = $this->xpath->query($xpath);
        $texts = [];
        foreach ($nodes as $node) {
            if ($node->nodeType === XML_TEXT_NODE) {
                $texts[] = trim($node->nodeValue);
            } else {
                $texts[] = trim($node->textContent);
            }
        }
        return $texts;
    }
    protected function getAttribute($xpath, $attribute) {
        $nodes = $this->xpath->query($xpath);
        $values = [];
        foreach ($nodes as $node) {
            if ($node->nodeType === XML_ELEMENT_NODE) {
                $values[] = $node->getAttribute($attribute);
            }
        }
        return $values;
    }
}

存储接口和实现

StorageInterface.php

<?php
namespace Crawler\Storage;
interface StorageInterface {
    public function save(array $data);
    public function get();
    public function clear();
    public function count();
}

DatabaseStorage.php

<?php
namespace Crawler\Storage;
class DatabaseStorage implements StorageInterface {
    protected $pdo;
    protected $table;
    public function __construct($host, $dbname, $user, $password, $table) {
        try {
            $this->pdo = new \PDO(
                "mysql:host=$host;dbname=$dbname;charset=utf8mb4",
                $user,
                $password,
                [\PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION]
            );
            $this->table = $table;
        } catch (\PDOException $e) {
            throw new \Exception("数据库连接失败: " . $e->getMessage());
        }
    }
    public function save(array $data) {
        $columns = array_keys($data);
        $placeholders = array_fill(0, count($columns), '?');
        $sql = "INSERT INTO " . $this->table . " (" . implode(', ', $columns) . ") 
                VALUES (" . implode(', ', $placeholders) . ")";
        try {
            $stmt = $this->pdo->prepare($sql);
            $stmt->execute(array_values($data));
            return $this->pdo->lastInsertId();
        } catch (\PDOException $e) {
            throw new \Exception("数据存储失败: " . $e->getMessage());
        }
    }
    public function get() {
        $stmt = $this->pdo->query("SELECT * FROM " . $this->table);
        return $stmt->fetchAll(\PDO::FETCH_ASSOC);
    }
    public function clear() {
        $this->pdo->query("DELETE FROM " . $this->table);
        return true;
    }
    public function count() {
        $stmt = $this->pdo->query("SELECT COUNT(*) FROM " . $this->table);
        return $stmt->fetchColumn();
    }
}

配置文件

config.php

<?php
return [
    'settings' => [
        'max_retries' => 3,
        'timeout' => 30,
        'concurrency' => 1,
        'use_random_user_agent' => true,
        'respect_robots_txt' => true
    ],
    'database' => [
        'host' => 'localhost',
        'dbname' => 'crawler',
        'user' => 'root',
        'password' => '',
        'table' => 'crawled_data'
    ],
    'output' => [
        'format' => 'json', // json, csv, database
        'path' => __DIR__ . '/data/'
    ],
    'selectors' => [
        'title' => [
            'selector' => 'h1',
            'attribute' => null // null表示获取文本
        ],
        'links' => [
            'selector' => 'a',
            'attribute' => 'href'
        ],
        'content' => [
            'selector' => '.content',
            'attribute' => null
        ]
    ]
];

使用示例

基本使用 (examples/basic.php)

<?php
require_once __DIR__ . '/vendor/autoload.php';
use Crawler\Core\Crawler;
use Crawler\Core\Request;
use Crawler\Parser\HTMLParser;
use Crawler\Storage\DatabaseStorage;
// 加载配置
$config = require __DIR__ . '/../config/config.php';
// 创建爬虫实例
$crawler = new Crawler($config['settings']);
// 设置请求
$request = new Request('https://example.com');
$request->setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64)');
$crawler->setRequest($request);
// 设置解析器
$parser = new HTMLParser($config['selectors']);
$crawler->setParser($parser);
// 设置存储
$storage = new DatabaseStorage(
    $config['database']['host'],
    $config['database']['dbname'],
    $config['database']['user'],
    $config['database']['password'],
    $config['database']['table']
);
$crawler->setStorage($storage);
// 设置回调
$crawler->setCallback('onSuccess', function($response) {
    echo "成功获取页面: " . $response->getUrl() . " (耗时: " . $response->getExecutionTime() . "s)\n";
});
$crawler->setCallback('onError', function($response) {
    echo "页面获取失败: " . $response->getUrl() . " (错误: " . $response->getError() . ")\n";
});
// 执行爬取
$response = $crawler->fetch('https://example.com');
// 获取统计信息
$stats = $crawler->getStats();
print_r($stats);

Composer 配置文件

composer.json

{
    "name": "mycrawler/mycrawler",
    "description": "PHP微爬虫框架",
    "type": "library",
    "require": {
        "php": ">=7.4",
        "ext-curl": "*",
        "ext-dom": "*",
        "ext-pdo": "*"
    },
    "autoload": {
        "psr-4": {
            "Crawler\\": "src/"
        }
    },
    "config": {
        "optimize-autoloader": true
    }
}

高级功能

多线程爬取 (ConcurrentCrawler)

<?php
namespace Crawler\Core;
use parallel\Runtime;
class ConcurrentCrawler {
    protected $config;
    protected $crawlers = [];
    public function __construct($config = []) {
        $this->config = array_merge([
            'max_crawlers' => 5,
            'max_requests_per_crawler' => 100
        ], $config);
    }
    public function crawl($urls) {
        $chunks = array_chunk($urls, ceil(count($urls) / $this->config['max_crawlers']));
        foreach ($chunks as $chunk) {
            $runtime = new Runtime();
            $this->crawlers[] = $runtime->run(function($urls, $config) {
                // 每个爬虫实例的代码
                require_once 'path/to/vendor/autoload.php';
                $crawler = new Crawler($config);
                foreach ($urls as $url) {
                    $crawler->fetch($url);
                }
            }, [$chunk, $this->config]);
        }
        // 等待所有爬虫完成
        foreach ($this->crawlers as $crawler) {
            $crawler->join();
        }
    }
}

测试文件

tests/UnitTest.php

<?php
use PHPUnit\Framework\TestCase;
use Crawler\Core\Crawler;
use Crawler\Core\Request;
class CrawlerTest extends TestCase {
    public function testFetchSuccess() {
        $crawler = new Crawler(['max_retries' => 3]);
        $request = new Request('https://example.com');
        $crawler->setRequest($request);
        $response = $crawler->fetch('https://example.com');
        $this->assertTrue($response->isSuccess());
        $this->assertNotEmpty($response->getBody());
    }
    public function testFetchFailure() {
        $crawler = new Crawler(['max_retries' => 1]);
        $response = $crawler->fetch('https://invalid.example.com');
        $this->assertNotTrue($response->isSuccess());
    }
}

运行命令

# 生成自动加载文件
composer dump-autoload
# 运行测试
vendor/bin/phpunit tests/
# 运行基本示例
php examples/basic.php

扩展建议

添加更多解析器：支持CSS选择器、XPath、JSONPath等
缓存功能：使用Redis/Memcached缓存已爬取的数据
断点续爬：记录爬取状态，支持中断后继续
数据清洗：添加数据过滤、去重、清洗功能
反爬处理：支持IP池、代理池、验证码识别等
队列系统：集成Redis队列实现任务分发
监控日志：添加详细的日志和监控功能

这个框架设计灵活，你可以根据实际需求进行扩展和调整，核心的Crawler类已经实现了基本功能,你可以在此基础上添加更多高级特性。

本文地址： https://www.bmjup.com/post/43607.html