IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

Java文件读写提速案例如何实操

wen java案例 29

本文目录导读：

Java文件读写提速案例如何实操

  1. 使用缓冲流（最基础实用的提速方法）
  2. 内存映射文件（超大文件最佳方案）
  3. 使用NIO批量读取
  4. 并行处理（多核CPU最佳实践）
  5. 最佳实践组合方案
  6. 性能对比测试代码

我来介绍几个实用的Java文件读写提速案例和具体操作步骤：

使用缓冲流（最基础实用的提速方法）

案例：大文件复制（100MB+）

// ❌ 慢速方式 - 不使用缓冲
try (FileInputStream fis = new FileInputStream("input.txt");
     FileOutputStream fos = new FileOutputStream("output.txt")) {
    int data;
    while ((data = fis.read()) != -1) {
        fos.write(data);
    }
}
// ✅ 快速方式 - 使用缓冲流
try (BufferedInputStream bis = new BufferedInputStream(
         new FileInputStream("input.txt"), 8192);  // 8KB缓冲区
     BufferedOutputStream bos = new BufferedOutputStream(
         new FileOutputStream("output.txt"), 8192)) {
    byte[] buffer = new byte[8192];
    int bytesRead;
    while ((bytesRead = bis.read(buffer)) != -1) {
        bos.write(buffer, 0, bytesRead);
    }
}
// 性能提升：10-100倍

内存映射文件（超大文件最佳方案）

案例：处理GB级日志文件

import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
public class MappedFileDemo {
    public static void main(String[] args) throws Exception {
        // 读取2GB的日志文件
        try (FileChannel channel = FileChannel.open(
                Paths.get("huge-log.txt"), 
                StandardOpenOption.READ)) {
            long fileSize = channel.size();
            // 分块映射，避免内存溢出
            long chunkSize = 100 * 1024 * 1024; // 100MB每块
            for (long position = 0; position < fileSize; position += chunkSize) {
                long size = Math.min(chunkSize, fileSize - position);
                MappedByteBuffer buffer = channel.map(
                    FileChannel.MapMode.READ_ONLY, position, size);
                // 直接操作内存中的数据，无需复制
                processData(buffer);
            }
        }
    }
    private static void processData(MappedByteBuffer buffer) {
        // 直接在内存中处理数据
        while (buffer.hasRemaining()) {
            byte b = buffer.get();
            // 处理每个字节...
        }
    }
}
// 性能提升：比传统IO快5-10倍

使用NIO批量读取

案例：CSV文件快速处理

import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public class NIOBatchReadDemo {
    public static List<String> fastReadLines(String filename) throws Exception {
        List<String> lines = new ArrayList<>();
        try (FileChannel channel = FileChannel.open(
                Path.of(filename), StandardOpenOption.READ)) {
            // 分配大缓冲区
            ByteBuffer buffer = ByteBuffer.allocateDirect(64 * 1024); // 64KB
            StringBuilder lineBuilder = new StringBuilder();
            while (channel.read(buffer) != -1) {
                buffer.flip();
                byte[] bytes = new byte[buffer.remaining()];
                buffer.get(bytes);
                // 处理当前批次的数据
                String chunk = new String(bytes, StandardCharsets.UTF_8);
                String[] chunkLines = chunk.split("\n", -1);
                for (int i = 0; i < chunkLines.length; i++) {
                    if (i == 0) {
                        // 拼接上一批次未完成的行
                        chunkLines[i] = lineBuilder + chunkLines[i];
                    }
                    if (i < chunkLines.length - 1) {
                        lines.add(chunkLines[i]);
                    } else {
                        // 最后一行可能不完整，保存到builder
                        lineBuilder = new StringBuilder(chunkLines[i]);
                    }
                }
                buffer.clear();
            }
            // 处理最后一行
            if (lineBuilder.length() > 0) {
                lines.add(lineBuilder.toString());
            }
        }
        return lines;
    }
}
// 性能提升：比BufferedReader快30-50%

并行处理（多核CPU最佳实践）

案例：快速统计JSON文件中的特定数据

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.atomic.LongAdder;
import java.util.stream.Stream;
public class ParallelFileProcessing {
    public static long countLines(String filename) throws Exception {
        LongAdder counter = new LongAdder();
        // 使用并行流处理
        try (Stream<String> lines = Files.lines(Path.of(filename))) {
            lines.parallel()
                 .filter(line -> line.contains("ERROR"))
                 .forEach(line -> {
                     // 处理匹配的行
                     counter.increment();
                     // 可以在这里添加更多的处理逻辑
                     // 注意线程安全
                 });
        }
        return counter.sum();
    }
    // 更复杂的分块并行处理
    public static void parallelChunkProcessing(String filename) throws Exception {
        long fileSize = Files.size(Path.of(filename));
        int chunkCount = Runtime.getRuntime().availableProcessors(); // CPU核心数
        long chunkSize = fileSize / chunkCount;
        Thread[] threads = new Thread[chunkCount];
        for (int i = 0; i < chunkCount; i++) {
            final int chunkIndex = i;
            long start = chunkIndex * chunkSize;
            long end = (chunkIndex == chunkCount - 1) ? 
                       fileSize : (chunkIndex + 1) * chunkSize;
            threads[i] = new Thread(() -> {
                processFileChunk(filename, start, end);
            });
            threads[i].start();
        }
        // 等待所有线程完成
        for (Thread thread : threads) {
            thread.join();
        }
    }
    private static void processFileChunk(String filename, long start, long end) {
        // 实现块处理逻辑
        System.out.printf("Processing chunk: %d - %d%n", start, end);
    }
}
// 性能提升：根据CPU核心数，可提升2-8倍

最佳实践组合方案

完整案例：高性能文件读取器

import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.util.function.Consumer;
public class HighPerformanceFileReader {
    private static final int BUFFER_SIZE = 64 * 1024; // 64KB
    private static final int DIRECT_BUFFER = true;
    /**
     * 高性能文件读取器
     * @param filename 文件名
     * @param consumer 数据处理回调
     */
    public static void fastRead(String filename, Consumer<String> consumer) 
            throws Exception {
        try (FileChannel channel = FileChannel.open(
                Path.of(filename), StandardOpenOption.READ)) {
            ByteBuffer buffer;
            if (DIRECT_BUFFER) {
                buffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
            } else {
                buffer = ByteBuffer.allocate(BUFFER_SIZE);
            }
            StringBuilder leftover = new StringBuilder();
            while (channel.read(buffer) != -1) {
                buffer.flip();
                // 转换为字符串
                byte[] bytes = new byte[buffer.remaining()];
                buffer.get(bytes);
                String data = leftover + new String(bytes, StandardCharsets.UTF_8);
                // 按行处理
                int lastNewLine = data.lastIndexOf('\n');
                if (lastNewLine >= 0) {
                    String completeData = data.substring(0, lastNewLine);
                    String[] lines = completeData.split("\n");
                    for (String line : lines) {
                        consumer.accept(line.trim());
                    }
                    leftover = new StringBuilder(data.substring(lastNewLine + 1));
                } else {
                    leftover.append(data);
                }
                buffer.clear();
            }
            // 处理最后剩余数据
            if (leftover.length() > 0) {
                consumer.accept(leftover.toString().trim());
            }
        }
    }
    // 使用示例
    public static void main(String[] args) throws Exception {
        long start = System.currentTimeMillis();
        fastRead("large-file.txt", line -> {
            // 处理每一行
            if (line.startsWith("ERROR")) {
                System.out.println("Found error: " + line);
            }
        });
        long end = System.currentTimeMillis();
        System.out.println("Processing time: " + (end - start) + "ms");
    }
}

性能对比测试代码

public class PerformanceTest {
    public static void main(String[] args) throws Exception {
        String testFile = "test-100mb.txt";
        int iterations = 5;
        // 测试不同方式
        for (int i = 0; i < iterations; i++) {
            System.out.println("=== Iteration " + (i + 1) + " ===");
            // 1. 传统FileReader
            long time1 = testMethod(
                () -> readWithFileReader(testFile));
            System.out.println("FileReader: " + time1 + "ms");
            // 2. BufferedReader
            long time2 = testMethod(
                () -> readWithBufferedReader(testFile));
            System.out.println("BufferedReader: " + time2 + "ms");
            // 3. NIO with buffer
            long time3 = testMethod(
                () -> readWithNIO(testFile));
            System.out.println("NIO Buffer: " + time3 + "ms");
            // 4. MappedByteBuffer
            long time4 = testMethod(
                () -> readWithMappedFile(testFile));
            System.out.println("MappedByteBuffer: " + time4 + "ms");
            System.out.println();
        }
    }
    private static long testMethod(Runnable method) throws Exception {
        long start = System.nanoTime();
        method.run();
        return (System.nanoTime() - start) / 1_000_000;
    }
    // 各方法的实现...
}
  1. 首选方案：BufferedReader + BufferedWriter，简单有效
  2. 大文件(>100MB)：使用MappedByteBuffer
  3. 追求极致性能：NIO + 直接缓冲区(allocateDirect)
  4. 多核CPU：并行处理（注意线程安全）
  5. 关键优化点：
    • 缓冲区大小：通常8-64KB
    • 使用直接缓冲区（Direct Buffer）避免数据拷贝
    • 批量处理代替逐字节处理

选择合适的方案可以轻松获得10-100倍的性能提升。

上一篇Java并发处理提速案例如何落地

下一篇Java网络请求提速案例怎么优化

相关文章

抱歉，评论功能暂时关闭!