IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

Java案例如何实现爬虫？

wen python案例 2

本文目录导读：

Java案例如何实现爬虫？

  1. 基础爬虫方案
  2. 高级爬虫框架 - WebMagic
  3. 使用HttpClient + Jsoup
  4. 数据持久化 - 保存到数据库
  5. 反爬虫处理
  6. 完整爬虫运行示例
  7. 注意事项

我来为你详细介绍Java实现爬虫的几种常见方案和完整案例。

基础爬虫方案

Maven依赖配置

<dependencies>
    <!-- HTTP客户端 -->
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.13</version>
    </dependency>
    <!-- HTML解析器 -->
    <dependency>
        <groupId>org.jsoup</groupId>
        <artifactId>jsoup</artifactId>
        <version>1.15.3</version>
    </dependency>
    <!-- JSON处理 -->
    <dependency>
        <groupId>com.google.code.gson</groupId>
        <artifactId>gson</artifactId>
        <version>2.9.0</version>
    </dependency>
</dependencies>

简单爬虫示例 - 抓取网页内容

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
public class SimpleCrawler {
    public static void main(String[] args) {
        String url = "https://example.com/news";
        crawl(url);
    }
    public static void crawl(String url) {
        try {
            // 发送HTTP请求并获取HTML文档
            Document document = Jsoup.connect(url)
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
                .timeout(5000)
                .get();
            // 解析标题
            String title = document.title();
            System.out.println("页面标题: " + title);
            // 提取所有链接
            Elements links = document.select("a[href]");
            for (Element link : links) {
                String href = link.attr("abs:href");
                String text = link.text();
                System.out.println("链接: " + text + " -> " + href);
            }
            // 提取特定元素
            Elements newsItems = document.select(".news-item");
            for (Element item : newsItems) {
                String title_text = item.select(".title").text();
                String desc = item.select(".description").text();
                System.out.println("新闻标题: " + title_text);
                System.out.println("描述: " + desc);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

高级爬虫框架 - WebMagic

添加依赖

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>0.7.5</version>
</dependency>
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>0.7.5</version>
</dependency>

WebMagic爬虫示例

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
public class WebMagicCrawler implements PageProcessor {
    // 配置爬虫设置
    private Site site = Site.me()
        .setRetryTimes(3)           // 重试次数
        .setSleepTime(1000)         // 请求间隔
        .setTimeOut(10000)          // 超时时间
        .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
    @Override
    public void process(Page page) {
        // 爬取列表页
        if (page.getUrl().regex("https://example.com/list/\\d+").match()) {
            // 添加分页链接到待爬取队列
            page.addTargetRequests(page.getHtml().links().regex("https://example.com/list/\\d+").all());
            // 添加详情页链接到待爬取队列
            page.addTargetRequests(page.getHtml().links().regex("https://example.com/detail/\\d+").all());
        }
        // 爬取详情页
        if (page.getUrl().regex("https://example.com/detail/\\d+").match()) {
            // 提取数据
            String title = page.getHtml().xpath("//h1[@class='title']/text()").get();
            String content = page.getHtml().xpath("//div[@class='content']/html()").get();
            String author = page.getHtml().css(".author", "text").get();
            String date = page.getHtml().xpath("//span[@class='date']/text()").get();
            // 保存数据到resultItems
            page.putField("title", title);
            page.putField("content", content);
            page.putField("author", author);
            page.putField("date", date);
            page.putField("url", page.getUrl().toString());
        }
    }
    @Override
    public Site getSite() {
        return site;
    }
    public static void main(String[] args) {
        // 启动爬虫
        Spider.create(new WebMagicCrawler())
            .addUrl("https://example.com/list/1")  // 起始URL
            .addPipeline(new ConsolePipeline())     // 控制台输出
            .addPipeline(new JsonFilePipeline("data")) // 保存为JSON文件
            .thread(5)                              // 线程数
            .run();                                 // 运行
    }
}

使用HttpClient + Jsoup

完整爬虫工具类

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AdvancedCrawler {
    private CloseableHttpClient httpClient;
    private Map<String, String> headers;
    public AdvancedCrawler() {
        this.httpClient = HttpClients.createDefault();
        this.headers = new HashMap<>();
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
        headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        headers.put("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
    }
    // 获取网页内容
    public String getPageContent(String url) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        // 设置请求头
        for (Map.Entry<String, String> entry : headers.entrySet()) {
            httpGet.setHeader(entry.getKey(), entry.getValue());
        }
        try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
            HttpEntity entity = response.getEntity();
            return EntityUtils.toString(entity, "UTF-8");
        }
    }
    // 解析并提取数据
    public void parseAndExtract(String html) {
        Document doc = Jsoup.parse(html);
        // 使用CSS选择器
        Elements elements = doc.select("div.article");
        for (Element element : elements) {
            String title = element.select("h2.title").text();
            String summary = element.select("p.summary").text();
            String link = element.select("a").attr("href");
            System.out.println("标题: " + title);
            System.out.println(" " + summary);
            System.out.println("链接: " + link);
        }
        // 使用正则表达式提取邮箱
        String text = doc.text();
        Pattern emailPattern = Pattern.compile("\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b");
        Matcher matcher = emailPattern.matcher(text);
        while (matcher.find()) {
            System.out.println("邮箱: " + matcher.group());
        }
    }
    // 带会话的爬取（处理登录等）
    public void crawlWithSession() throws IOException {
        // 实现登录并维持会话
        CloseableHttpClient sessionClient = HttpClients.custom()
            .setDefaultRequestConfig(null)
            .build();
        // 添加Cookie处理
        // 实际使用时需要添加CookieStore
    }
    public static void main(String[] args) {
        AdvancedCrawler crawler = new AdvancedCrawler();
        try {
            String content = crawler.getPageContent("https://example.com");
            crawler.parseAndExtract(content);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

数据持久化 - 保存到数据库

import java.sql.*;
import java.util.List;
import java.util.ArrayList;
public class DataPersistence {
    private Connection connection;
    public DataPersistence() {
        // 初始化数据库连接
        try {
            Class.forName("com.mysql.cj.jdbc.Driver");
            connection = DriverManager.getConnection(
                "jdbc:mysql://localhost:3306/crawler",
                "username",
                "password"
            );
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    // 保存爬取的数据
    public void saveArticle(String title, String content, String url) {
        String sql = "INSERT INTO articles (title, content, url, crawl_time) VALUES (?, ?, ?, NOW())";
        try (PreparedStatement pstmt = connection.prepareStatement(sql)) {
            pstmt.setString(1, title);
            pstmt.setString(2, content);
            pstmt.setString(3, url);
            pstmt.executeUpdate();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }
    // 批量保存
    public void batchSave(List<Article> articles) {
        String sql = "INSERT INTO articles (title, content, url) VALUES (?, ?, ?)";
        try (PreparedStatement pstmt = connection.prepareStatement(sql)) {
            for (Article article : articles) {
                pstmt.setString(1, article.getTitle());
                pstmt.setString(2, article.getContent());
                pstmt.setString(3, article.getUrl());
                pstmt.addBatch();
            }
            pstmt.executeBatch();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }
    // 文章实体类
    static class Article {
        private String title;
        private String content;
        private String url;
        // getters and setters
        public String getTitle() { return title; }
        public void setTitle(String title) { this.title = title; }
        public String getContent() { return content; }
        public void setContent(String content) { this.content = content; }
        public String getUrl() { return url; }
        public void setUrl(String url) { this.url = url; }
    }
}

反爬虫处理

import java.util.Random;
public class AntiCrawlStrategy {
    private static Random random = new Random();
    // 随机User-Agent
    private static String[] USER_AGENTS = {
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
        "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15",
        "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15"
    };
    // 获取随机User-Agent
    public static String getRandomUserAgent() {
        return USER_AGENTS[random.nextInt(USER_AGENTS.length)];
    }
    // 添加请求延迟
    public static void delay() {
        try {
            // 随机延迟1-3秒
            Thread.sleep(1000 + random.nextInt(2000));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    // 使用代理IP
    public static String getProxyIP() {
        // 实际使用时从代理池获取
        return "http://proxy-ip:port";
    }
}

完整爬虫运行示例

import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class CrawlerRunner {
    public static void main(String[] args) {
        // 使用线程池控制并发
        ExecutorService executor = Executors.newFixedThreadPool(5);
        // 爬取多个URL
        String[] urls = {
            "https://example.com/page1",
            "https://example.com/page2",
            "https://example.com/page3"
        };
        for (String url : urls) {
            executor.submit(() -> {
                try {
                    // 添加延迟避免被封
                    Thread.sleep(1000);
                    // 执行爬取
                    AdvancedCrawler crawler = new AdvancedCrawler();
                    String content = crawler.getPageContent(url);
                    crawler.parseAndExtract(content);
                    System.out.println("成功爬取: " + url);
                } catch (Exception e) {
                    System.err.println("爬取失败: " + url);
                    e.printStackTrace();
                }
            });
        }
        executor.shutdown();
    }
}

注意事项

  1. 遵守robots.txt协议
  2. 设置合理的请求频率
  3. 处理异常和重试机制
  4. 数据去重和增量爬取
  5. 遵守网站使用条款
  6. 注意数据隐私和版权问题

这个案例覆盖了从简单到复杂的Java爬虫实现，你可以根据实际需求选择合适的方案,爬虫技术要合法使用。

上一篇HttpClient怎么用？

下一篇Java案例如何实现网络请求？

相关文章

抱歉，评论功能暂时关闭!