本文目录导读：
我来为你提供几种PHP自动翻页抓取的实现方案：
基于cURL的通用翻页抓取
<?php
class PageCrawler {
private $ch;
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
private $maxRetries = 3;
private $timeout = 30;
public function __construct() {
$this->ch = curl_init();
$this->setCurlOptions();
}
private function setCurlOptions() {
curl_setopt_array($this->ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_ENCODING => 'gzip, deflate',
CURLOPT_HTTPHEADER => [
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8',
'Cache-Control: no-cache',
'Pragma: no-cache'
]
]);
}
/**
* 抓取单个页面
*/
public function fetchPage($url) {
for ($retry = 0; $retry < $this->maxRetries; $retry++) {
curl_setopt($this->ch, CURLOPT_URL, $url);
$content = curl_exec($this->ch);
if ($content !== false) {
return $content;
}
// 等待后重试
sleep(2);
}
return false;
}
/**
* 自动翻页抓取（基于URL模式）
*/
public function crawlByUrlPattern($baseUrl, $startPage = 1, $maxPages = 10, $urlPattern = null) {
$results = [];
for ($page = $startPage; $page <= $maxPages; $page++) {
// URL模式：支持{page}占位符
if ($urlPattern) {
$url = str_replace('{page}', $page, $urlPattern);
} else {
// 默认页码格式
$separator = strpos($baseUrl, '?') !== false ? '&' : '?';
$url = $baseUrl . $separator . 'page=' . $page;
}
echo "抓取第 {$page} 页: {$url}\n";
$content = $this->fetchPage($url);
if ($content === false) {
echo "第 {$page} 页抓取失败\n";
break;
}
$parsedData = $this->parseContent($content);
if (empty($parsedData)) {
echo "第 {$page} 页无数据，停止翻页\n";
break;
}
$results = array_merge($results, $parsedData);
// 避免请求过快
usleep(500000); // 0.5秒
}
return $results;
}
/**
* 基于正则表达式解析内容
*/
private function parseContent($content) {
$data = [];
// 示例：解析所有标题和链接
preg_match_all('/<h2[^>]*>\s*<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>\s*<\/h2>/s', $content, $matches, PREG_SET_ORDER);
foreach ($matches as $match) {
$data[] = [
'url' => $match[1],
'title' => trim(strip_tags($match[2]))
];
}
return $data;
}
public function __destruct() {
curl_close($this->ch);
}
}
// 使用示例
$crawler = new PageCrawler();
$results = $crawler->crawlByUrlPattern(
'https://example.com/list',
1,
20,
'https://example.com/list?page={page}'
);
print_r($results);
?>
基于DOM解析的翻页抓取
<?php
class DOMCrawler {
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
/**
* 使用DOMDocument解析内容
*/
public function crawl($startUrl, $maxPages = 10) {
$results = [];
$currentUrl = $startUrl;
$pageCount = 0;
while ($currentUrl && $pageCount < $maxPages) {
$pageCount++;
echo "抓取页面 {$pageCount}: {$currentUrl}\n";
// 获取页面内容
$html = $this->fetchContent($currentUrl);
if ($html === false) {
break;
}
// DOM解析
$dom = new DOMDocument();
@$dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'));
$xpath = new DOMXPath($dom);
// 解析列表内容
$items = $xpath->query('//div[@class="list-item"]');
foreach ($items as $item) {
$titleNode = $xpath->query('.//h2/a', $item)->item(0);
$title = $titleNode ? $titleNode->nodeValue : '';
$link = $titleNode ? $titleNode->getAttribute('href') : '';
$dates = $xpath->query('.//span[@class="date"]', $item);
$date = $dates->length > 0 ? $dates->item(0)->nodeValue : '';
$results[] = [
'title' => trim($title),
'link' => $link,
'date' => trim($date)
];
}
// 查找下一页链接
$nextLink = $xpath->query('//a[contains(@class, "next") or contains(text(), "下一页") or contains(text(), ">")]');
$currentUrl = null;
if ($nextLink->length > 0) {
$href = $nextLink->item(0)->getAttribute('href');
$currentUrl = $this->resolveUrl($href, $currentUrl);
}
// 防爬虫延时
sleep(1);
}
return $results;
}
private function fetchContent($url) {
$ch = curl_init();
curl_setopt_array($ch, [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_TIMEOUT => 30
]);
$content = curl_exec($ch);
curl_close($ch);
return $content;
}
/**
* 解析相对URL为绝对URL
*/
private function resolveUrl($href, $baseUrl) {
if (strpos($href, 'http') === 0) {
return $href;
}
if (strpos($href, '//') === 0) {
return 'http:' . $href;
}
if (strpos($href, '/') === 0) {
$parts = parse_url($baseUrl);
return $parts['scheme'] . '://' . $parts['host'] . $href;
}
return dirname($baseUrl) . '/' . $href;
}
}
// 使用示例
$crawler = new DOMCrawler();
$data = $crawler->crawl('https://example.com/news/', 15);
print_r($data);
?>
支持AJAX加载的翻页抓取
<?php
class AjaxCrawler {
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
/**
* 爬取AJAX数据
*/
public function crawlAjax($apiUrl, $params = [], $maxPages = 10) {
$results = [];
$page = 1;
while ($page <= $maxPages) {
// 动态参数
$params['page'] = $page;
$params['pageSize'] = 20;
$params['timestamp'] = time();
$url = $apiUrl . '?' . http_build_query($params);
echo "AJAX请求第 {$page} 页: {$url}\n";
$response = $this->fetchAjaxData($url);
if ($response === false) {
break;
}
// 解析JSON数据
$data = json_decode($response, true);
if (empty($data['data']['list'])) {
echo "第 {$page} 页无数据\n";
break;
}
// 合并结果
$results = array_merge($results, $data['data']['list']);
// 检查是否还有下一页
if ($data['data']['hasNext'] === false || $page >= $data['data']['totalPages']) {
break;
}
$page++;
usleep(500000);
}
return $results;
}
private function fetchAjaxData($url) {
$ch = curl_init();
// 设置AJAX请求头
$headers = [
'X-Requested-With: XMLHttpRequest',
'Accept: application/json, text/javascript, */*; q=0.01',
'Content-Type: application/x-www-form-urlencoded; charset=UTF-8',
'Referer: ' . $this->getReferer($url)
];
curl_setopt_array($ch, [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_HTTPHEADER => $headers,
CURLOPT_TIMEOUT => 30,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_ENCODING => 'gzip, deflate'
]);
$content = curl_exec($ch);
curl_close($ch);
return $content;
}
private function getReferer($url) {
$parts = parse_url($url);
return $parts['scheme'] . '://' . $parts['host'];
}
}
// 使用示例
$crawler = new AjaxCrawler();
$data = $crawler->crawlAjax(
'https://api.example.com/news/list',
['category' => 'sports', 'sort' => 'latest'],
15
);
print_r($data);
?>
分布式/多线程翻页抓取
<?php
class MultiThreadCrawler {
private $maxThreads = 5;
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
/**
* 并发抓取多页
*/
public function crawlConcurrent($baseUrl, $startPage = 1, $endPage = 20) {
$results = [];
$threadPool = [];
$pages = range($startPage, $endPage);
foreach ($pages as $page) {
$threadPool[] = $page;
if (count($threadPool) >= $this->maxThreads) {
$results = array_merge($results, $this->fetchPagesConcurrent($threadPool, $baseUrl));
$threadPool = [];
sleep(1); // 避免请求过快
}
}
if (!empty($threadPool)) {
$results = array_merge($results, $this->fetchPagesConcurrent($threadPool, $baseUrl));
}
return $results;
}
private function fetchPagesConcurrent($pages, $baseUrl) {
$handles = [];
$contents = [];
$mh = curl_multi_init();
foreach ($pages as $i => $page) {
$ch = curl_init();
$url = str_replace('{page}', $page, $baseUrl);
curl_setopt_array($ch, [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_TIMEOUT => 30,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_ENCODING => 'gzip, deflate'
]);
curl_multi_add_handle($mh, $ch);
$handles[$i] = $ch;
}
// 执行请求
$active = null;
do {
$status = curl_multi_exec($mh, $active);
if ($active) {
curl_multi_select($mh);
}
} while ($active && $status == CURLM_OK);
// 收集结果
foreach ($handles as $i => $ch) {
$contents[$i] = curl_multi_getcontent($ch);
curl_multi_remove_handle($mh, $ch);
curl_close($ch);
}
curl_multi_close($mh);
// 解析内容
$results = [];
foreach ($contents as $content) {
if ($content) {
$results = array_merge($results, $this->parseContent($content));
}
}
return $results;
}
private function parseContent($html) {
preg_match_all('/<h2[^>]*>\s*<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>\s*<\/h2>/s', $html, $matches, PREG_SET_ORDER);
$data = [];
foreach ($matches as $match) {
$data[] = [
'url' => $match[1],
'title' => trim(strip_tags($match[2]))
];
}
return $data;
}
}
// 使用示例
$crawler = new MultiThreadCrawler();
$urlPattern = 'https://example.com/list?page={page}';
$data = $crawler->crawlConcurrent($urlPattern, 1, 30);
print_r($data);
?>
使用建议
-
反爬虫处理：
- 设置合理的User-Agent
- 控制请求频率（添加延时）
- 使用代理IP池
- 模拟浏览器行为
-
数据解析：
- 优先使用DOMDocument/XPath
- 正则表达式适合简单提取
- 注意编码问题
-
错误处理：
- 设置重试机制
- 记录日志
- 异常捕获
-
法律合规：
- 遵守robots.txt
- 不要高频请求
- 尊重版权
选择方案时，根据目标网站的实现方式来决定：静态页面用第一种，动态加载用第三种,需要高并发用第四种。
