PHP 爬虫数据清洗

本文目录导读：

1. 基础数据清洗框架
2. 高级清洗工具类
3. 爬虫数据清洗实战示例
4. 数据验证和异常处理
5. 缓存和存储优化
6. 完整使用示例
7. 性能优化建议

我来为你详细讲解PHP爬虫数据清洗的完整方案,包括常用技术和最佳实践。

基础数据清洗框架

<?php
class DataCleaner {
    private $data;
    private $errors = [];
    public function __construct($data) {
        $this->data = $data;
    }
    // 主清洗流程
    public function clean() {
        $this->removeDuplicate();
        $this->removeEmpty();
        $this->formatText();
        $this->formatDate();
        $this->formatNumber();
        $this->filterSpecialChars();
        return [
            'data' => $this->data,
            'errors' => $this->errors
        ];
    }
    // 去重
    private function removeDuplicate() {
        $this->data = array_unique($this->data, SORT_REGULAR);
    }
    // 移空值
    private function removeEmpty() {
        foreach ($this->data as $key => $value) {
            if (empty($value) && $value !== '0') {
                unset($this->data[$key]);
            }
        }
    }
    // 格式化文本
    private function formatText() {
        foreach ($this->data as $key => $value) {
            if (is_string($value)) {
                // 去除多余空格
                $this->data[$key] = trim(preg_replace('/\s+/', ' ', $value));
                // 去除HTML标签
                $this->data[$key] = strip_tags($this->data[$key]);
                // 转换特殊字符
                $this->data[$key] = htmlspecialchars($this->data[$key]);
            }
        }
    }
    // 格式化日期
    private function formatDate() {
        foreach ($this->data as $key => $value) {
            if (is_string($value) && $this->isDate($value)) {
                $date = date_create($value);
                if ($date) {
                    $this->data[$key] = $date->format('Y-m-d H:i:s');
                }
            }
        }
    }
    // 格式化数字
    private function formatNumber() {
        foreach ($this->data as $key => $value) {
            if (is_numeric($value)) {
                $this->data[$key] = floatval($value);
            }
        }
    }
    // 过滤特殊字符
    private function filterSpecialChars() {
        $pattern = '/[^\w\s\x{4e00}-\x{9fa5}，。！？、；：""''（）《》@#$%^&*()-+=,.!?;:\/\\|]/u';
        foreach ($this->data as $key => $value) {
            if (is_string($value)) {
                $this->data[$key] = preg_replace($pattern, '', $value);
            }
        }
    }
    // 判断是否为日期
    private function isDate($value) {
        try {
            new DateTime($value);
            return true;
        } catch (Exception $e) {
            return false;
        }
    }
}

高级清洗工具类

<?php
class AdvancedCleaner {
    // 提取图片URL
    public function extractImages($html) {
        $pattern = '/<img[^>]+src=["\']([^"\']+)["\']/i';
        preg_match_all($pattern, $html, $matches);
        return $matches[1] ?? [];
    }
    // 提取链接
    public function extractLinks($html) {
        $pattern = '/<a[^>]+href=["\']([^"\']+)["\']/i';
        preg_match_all($pattern, $html, $matches);
        return $matches[1] ?? [];
    }
    // 清洗HTML
    public function cleanHtml($html, $allowedTags = []) {
        if (empty($allowedTags)) {
            $allowedTags = '<p><br><b><strong><i><em><ul><li><ol><h1><h2><h3><h4><h5>';
        }
        return strip_tags($html, $allowedTags);
    }
    // 格式化URL
    public function normalizeUrl($url, $baseUrl = '') {
        if (filter_var($url, FILTER_VALIDATE_URL)) {
            return $url;
        }
        if (strpos($url, 'http') === 0) {
            return $url;
        }
        if (!empty($baseUrl)) {
            $base = parse_url($baseUrl);
            $scheme = $base['scheme'] ?? 'http';
            $host = $base['host'] ?? '';
            $path = $base['path'] ?? '';
            if (strpos($url, '/') === 0) {
                return "{$scheme}://{$host}{$url}";
            }
            return "{$scheme}://{$host}" . dirname($path) . "/{$url}";
        }
        return $url;
    }
    // 清洗JSON数据
    public function cleanJson($jsonData) {
        $data = json_decode($jsonData, true);
        if (json_last_error() !== JSON_ERROR_NONE) {
            // 尝试修复JSON
            $data = $this->repairJson($jsonData);
        }
        return $data;
    }
    // 修复损坏的JSON
    private function repairJson($json) {
        // 添加缺失的引号
        $json = preg_replace('/([{,])(\s*)([A-Za-z0-9_]+)\s*:/', '$1"$3":', $json);
        // 确保是有效JSON
        return json_decode($json, true);
    }
    // 清洗CSV数据
    public function cleanCsv($csvContent) {
        $rows = str_getcsv($csvContent, "\n");
        $cleanData = [];
        foreach ($rows as $row) {
            $fields = str_getcsv($row, ',');
            $cleanFields = array_map('trim', $fields);
            $cleanData[] = $cleanFields;
        }
        return $cleanData;
    }
    // 格式转换
    public function convertEncoding($data, $from = 'GB2312', $to = 'UTF-8') {
        if (is_array($data)) {
            return array_map(function($item) use ($from, $to) {
                return $this->convertEncoding($item, $from, $to);
            }, $data);
        }
        if (is_string($data)) {
            $converted = @iconv($from, $to . '//IGNORE', $data);
            if ($converted !== false) {
                return $converted;
            }
            // iconv失败时使用mb_convert_encoding
            if (function_exists('mb_convert_encoding')) {
                return mb_convert_encoding($data, $to, $from);
            }
        }
        return $data;
    }
}

爬虫数据清洗实战示例

<?php
// 爬虫数据清洗完整示例
class CrawlerDataProcessor {
    private $rawData;
    private $cleaner;
    private $advancedCleaner;
    public function __construct($rawData) {
        $this->rawData = $rawData;
        $this->cleaner = new DataCleaner($rawData);
        $this->advancedCleaner = new AdvancedCleaner();
    }
    public function process() {
        $processed = [];
        foreach ($this->rawData as $item) {
            $cleanItem = [];
            // 处理标题
            if (isset($item['title'])) {
                $cleanItem['title'] = $this->cleanText($item['title']);
            }
            // 处理内容
            if (isset($item['content'])) {
                $cleanItem['content'] = $this->cleanContent($item['content']);
            }
            // 处理URL
            if (isset($item['url'])) {
                $cleanItem['url'] = $this->cleanUrl($item['url']);
            }
            // 处理图片
            if (isset($item['images']) && is_array($item['images'])) {
                $cleanItem['images'] = array_map(function($img) {
                    return $this->cleanUrl($img);
                }, $item['images']);
            }
            // 处理日期
            if (isset($item['date'])) {
                $cleanItem['date'] = $this->cleanDate($item['date']);
            }
            // 处理价格
            if (isset($item['price'])) {
                $cleanItem['price'] = $this->cleanPrice($item['price']);
            }
            $processed[] = $cleanItem;
        }
        // 最终清洗
        $result = $this->cleaner->clean();
        return [
            'data' => $processed,
            'count' => count($processed),
            'errors' => $result['errors']
        ];
    }
    // 清洗文本
    private function cleanText($text) {
        // 移除HTML标签
        $text = strip_tags($text);
        // 移除多余空格
        $text = preg_replace('/\s+/', ' ', $text);
        // 移除控制字符
        $text = preg_replace('/[\x00-\x1F\x7F]/', '', $text);
        // 转换编码
        $text = mb_convert_encoding($text, 'UTF-8', 'auto');
        return trim($text);
    }
    // 清洗内容
    private function cleanContent($content) {
        // 保留基本HTML标签
        $allowedTags = '<p><br><b><strong><i><em><ul><li><ol><h1><h2><h3><h4><h5>';
        $content = strip_tags($content, $allowedTags);
        // 移除script和style
        $content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
        $content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
        // 移除注释
        $content = preg_replace('/<!--.*?-->/s', '', $content);
        return $content;
    }
    // 清洗URL
    private function cleanUrl($url) {
        $url = html_entity_decode($url);
        $url = trim($url);
        // 检查URL合法性
        if (!filter_var($url, FILTER_VALIDATE_URL)) {
            // 尝试修复相对路径
            $url = $this->advancedCleaner->normalizeUrl($url);
        }
        // 移除JavaScript链接
        if (preg_match('/^javascript:/i', $url)) {
            return '';
        }
        return $url;
    }
    // 清洗日期
    private function cleanDate($date) {
        // 处理各种日期格式
        $patterns = [
            '/(\d{4})年(\d{1,2})月(\d{1,2})日/',
            '/(\d{4})-(\d{1,2})-(\d{1,2})/',
            '/(\d{4})\/(\d{1,2})\/(\d{1,2})/'
        ];
        foreach ($patterns as $pattern) {
            if (preg_match($pattern, $date, $matches)) {
                return sprintf("%04d-%02d-%02d", $matches[1], $matches[2], $matches[3]);
            }
        }
        // 尝试直接解析
        $timestamp = strtotime($date);
        if ($timestamp !== false) {
            return date('Y-m-d', $timestamp);
        }
        return null;
    }
    // 清洗价格
    private function cleanPrice($price) {
        // 移除货币符号和空格
        $price = preg_replace('/[^\d.]/', '', $price);
        return floatval($price);
    }
}

数据验证和异常处理

<?php
class DataValidator {
    public function validate($data, $rules) {
        $errors = [];
        foreach ($rules as $field => $fieldRules) {
            $value = $data[$field] ?? null;
            foreach ($fieldRules as $rule) {
                $result = $this->validateRule($field, $value, $rule);
                if ($result !== true) {
                    $errors[$field][] = $result;
                }
            }
        }
        return $errors;
    }
    private function validateRule($field, $value, $rule) {
        if (is_string($rule)) {
            return $this->applyRule($field, $value, $rule);
        }
        if (is_array($rule)) {
            $type = $rule[0] ?? '';
            $params = $rule[1] ?? null;
            return $this->applyRule($field, $value, $type, $params);
        }
        return true;
    }
    private function applyRule($field, $value, $rule, $params = null) {
        switch ($rule) {
            case 'required':
                return !empty($value) ? true : "$field is required";
            case 'email':
                return filter_var($value, FILTER_VALIDATE_EMAIL) ? true : "$field is not a valid email";
            case 'url':
                return filter_var($value, FILTER_VALIDATE_URL) ? true : "$field is not a valid URL";
            case 'numeric':
                return is_numeric($value) ? true : "$field must be numeric";
            case 'min_length':
                return strlen($value) >= $params ? true : "$field must be at least $params characters";
            case 'max_length':
                return strlen($value) <= $params ? true : "$field must be no more than $params characters";
            case 'regex':
                return preg_match($params, $value) ? true : "$field doesn't match the pattern";
            default:
                return true;
        }
    }
}

缓存和存储优化

<?php
class DataStorage {
    private $connection;
    public function __construct($host, $user, $pass, $dbname) {
        $this->connection = new PDO(
            "mysql:host=$host;dbname=$dbname;charset=utf8mb4",
            $user,
            $pass,
            [
                PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
                PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC
            ]
        );
    }
    // 批量插入
    public function batchInsert($table, $data) {
        $sql = "INSERT INTO $table (";
        $columns = array_keys($data[0]);
        $sql .= implode(', ', $columns) . ") VALUES ";
        $values = [];
        foreach ($data as $item) {
            $placeholders = array_fill(0, count($columns), '?');
            $values = array_merge($values, array_values($item));
            $sql .= "(" . implode(', ', $placeholders) . "), ";
        }
        $sql = rtrim($sql, ', ');
        $stmt = $this->connection->prepare($sql);
        $stmt->execute($values);
        return $stmt->rowCount();
    }
    // 增量更新
    public function upsert($table, $data, $uniqueFields) {
        $columns = array_keys($data);
        $sql = "INSERT INTO $table (" . implode(', ', $columns) . ") VALUES (";
        $placeholders = array_fill(0, count($columns), '?');
        $sql .= implode(', ', $placeholders) . ") ";
        // 添加ON DUPLICATE KEY UPDATE
        $updates = [];
        foreach ($columns as $column) {
            if (!in_array($column, $uniqueFields)) {
                $updates[] = "$column = VALUES($column)";
            }
        }
        $sql .= "ON DUPLICATE KEY UPDATE " . implode(', ', $updates);
        $stmt = $this->connection->prepare($sql);
        $stmt->execute(array_values($data));
        return $stmt->rowCount();
    }
}

完整使用示例

<?php
// 使用示例
$rawData = [
    [
        'title' => '  Apple iPhone 15 Pro   ',
        'content' => '<p>最新款<i>智能手机</i></p><script>alert("xss")</script>',
        'url' => '/products/iphone-15-pro',
        'price' => '¥9,999.00',
        'date' => '2024年1月15日',
        'images' => ['http://example.com/img1.jpg', '/img2.jpg']
    ],
    // ... 更多数据
];
// 创建处理器
$processor = new CrawlerDataProcessor($rawData);
// 处理数据
$result = $processor->process();
// 验证数据
$validator = new DataValidator();
$validationErrors = $validator->validate($result['data'], [ => ['required', ['min_length', 10]],
    'url' => ['url'],
    'price' => ['numeric']
]);
// 输出结果
echo "处理数据数量: " . $result['count'] . "\n";
if (!empty($validationErrors)) {
    echo "验证错误:\n";
    print_r($validationErrors);
} else {
    echo "数据验证通过!\n";
}
// 存储数据
$storage = new DataStorage('localhost', 'user', 'password', 'crawler');
foreach (array_chunk($result['data'], 100) as $chunk) { // 分批插入
    $storage->batchInsert('products', $chunk);
}

性能优化建议

1. 使用批量处理：分批处理数据，避免内存溢出
2. 过滤无效数据：在早期阶段就过滤掉无用数据
3. 使用缓存：对常用数据使用内存缓存
4. 并行处理：使用多进程处理大量数据
5. 合理的正则表达式：避免过度复杂的正则表达式

这个完整的PHP爬虫数据清洗方案覆盖了数据清洗的各个方面，包括基础清洗、高级处理、验证和存储,你可以根据实际需求调整和扩展这些功能。