怎么用脚本拆分大文件

本文目录导读：

1. Linux/Mac 命令行split工具
2. Python脚本拆分
3. 使用示例
4. 合并拆分的文件
5. 注意事项

我来介绍几种拆分大文件的方法,包括命令行和Python脚本。

Linux/Mac 命令行split工具

按大小拆分

# 每个文件100MB
split -b 100M large_file.txt part_
# 每个文件100MB，使用数字后缀
split -b 100M -d large_file.txt part_
# 每个文件100MB，数字后缀，指定后缀长度
split -b 100m -d -a 3 large_file.txt part_

按行数拆分

# 每10000行一个文件
split -l 10000 large_file.txt part_
# 每10000行，数字后缀
split -l 10000 -d large_file.txt part_

Python脚本拆分

按文件大小拆分

#!/usr/bin/env python3
# split_by_size.py
import os
def split_file_by_size(input_file, output_prefix, chunk_size_mb):
    """按指定大小拆分文件"""
    chunk_size = chunk_size_mb * 1024 * 1024  # 转换为字节
    file_count = 0
    bytes_written = 0
    with open(input_file, 'rb') as infile:
        while True:
            chunk = infile.read(chunk_size)
            if not chunk:
                break
            output_file = f"{output_prefix}_{file_count:03d}.part"
            with open(output_file, 'wb') as outfile:
                outfile.write(chunk)
            file_count += 1
            bytes_written += len(chunk)
            print(f"已创建: {output_file} ({len(chunk)} bytes)")
    print(f"\n完成！共拆分 {file_count} 个文件，总大小 {bytes_written / 1024 / 1024:.2f} MB")
if __name__ == "__main__":
    import sys
    if len(sys.argv) != 4:
        print("用法: python split_by_size.py <输入文件> <输出前缀> <大小MB>")
        sys.exit(1)
    split_file_by_size(sys.argv[1], sys.argv[2], int(sys.argv[3]))

按行数拆分

#!/usr/bin/env python3
# split_by_lines.py
def split_file_by_lines(input_file, output_prefix, lines_per_file):
    """按行数拆分文件"""
    file_count = 0
    line_count = 0
    current_file = None
    with open(input_file, 'r', encoding='utf-8') as infile:
        for line in infile:
            if line_count % lines_per_file == 0:
                if current_file:
                    current_file.close()
                output_file = f"{output_prefix}_{file_count:03d}.txt"
                current_file = open(output_file, 'w', encoding='utf-8')
                file_count += 1
                print(f"正在写入: {output_file}")
            current_file.write(line)
            line_count += 1
    if current_file:
        current_file.close()
    print(f"\n完成！共拆分 {file_count} 个文件，总行数 {line_count}")
if __name__ == "__main__":
    import sys
    if len(sys.argv) != 4:
        print("用法: python split_by_lines.py <输入文件> <输出前缀> <行数>")
        sys.exit(1)
    split_file_by_lines(sys.argv[1], sys.argv[2], int(sys.argv[3]))

通用拆分工具（支持二进制和文本）

#!/usr/bin/env python3
# split_tool.py
import os
import argparse
from pathlib import Path
class FileSplitter:
    def __init__(self, input_file, output_dir=None, buffer_size=1024*1024):
        self.input_file = Path(input_file)
        self.output_dir = output_dir or self.input_file.parent
        self.buffer_size = buffer_size
        self.file_size = self.input_file.stat().st_size
        os.makedirs(self.output_dir, exist_ok=True)
    def split_by_size(self, chunk_size_mb):
        """按大小拆分"""
        chunk_size = chunk_size_mb * 1024 * 1024
        parts = 0
        with open(self.input_file, 'rb') as infile:
            while True:
                chunk = infile.read(chunk_size)
                if not chunk:
                    break
                output_file = self.output_dir / f"{self.input_file.stem}_part{parts:03d}{self.input_file.suffix}"
                with open(output_file, 'wb') as outfile:
                    outfile.write(chunk)
                parts += 1
                print(f"拆分第 {parts} 部分: {output_file}")
        return parts
    def split_by_lines(self, lines_per_file):
        """按行拆分（文本文件）"""
        parts = 0
        line_count = 0
        current_file = None
        with open(self.input_file, 'r', encoding='utf-8') as infile:
            for line in infile:
                if line_count % lines_per_file == 0:
                    if current_file:
                        current_file.close()
                    output_file = self.output_dir / f"{self.input_file.stem}_part{parts:03d}.txt"
                    current_file = open(output_file, 'w', encoding='utf-8')
                    parts += 1
                current_file.write(line)
                line_count += 1
        if current_file:
            current_file.close()
        return parts
    def split_by_count(self, num_parts):
        """按数量均分"""
        chunk_size = self.file_size // num_parts
        parts = 0
        with open(self.input_file, 'rb') as infile:
            for i in range(num_parts):
                if i == num_parts - 1:  # 最后一个文件包含剩余所有内容
                    chunk = infile.read()
                else:
                    chunk = infile.read(chunk_size)
                if not chunk:
                    break
                output_file = self.output_dir / f"{self.input_file.stem}_part{i:03d}{self.input_file.suffix}"
                with open(output_file, 'wb') as outfile:
                    outfile.write(chunk)
                parts += 1
        return parts
# 命令行接口
def main():
    parser = argparse.ArgumentParser(description='拆分大文件工具')
    parser.add_argument('input', help='输入文件路径')
    parser.add_argument('-o', '--output-dir', help='输出目录（默认：输入文件所在目录）')
    parser.add_argument('-s', '--size', type=float, help='按大小拆分（MB）')
    parser.add_argument('-l', '--lines', type=int, help='按行数拆分（文本文件）')
    parser.add_argument('-n', '--num-parts', type=int, help='按数量均分')
    args = parser.parse_args()
    splitter = FileSplitter(args.input, args.output_dir)
    if args.size:
        parts = splitter.split_by_size(args.size)
        print(f"按大小拆分完成，共 {parts} 个文件")
    elif args.lines:
        parts = splitter.split_by_lines(args.lines)
        print(f"按行数拆分完成，共 {parts} 个文件")
    elif args.num_parts:
        parts = splitter.split_by_count(args.num_parts)
        print(f"按数量拆分完成，共 {parts} 个文件")
    else:
        parser.print_help()
if __name__ == "__main__":
    main()

使用示例

命令行使用

# Linux/Mac
split -b 100m large_file.zip part_
split -l 10000 data.csv data_part_
# Python脚本使用
python split_tool.py large_file.zip -s 100 -o output_dir
python split_tool.py data.csv -l 10000 -o output_dir
python split_tool.py video.mp4 -n 5 -o output_dir

合并拆分的文件

#!/usr/bin/env python3
# merge_files.py
import os
import sys
def merge_files(input_pattern, output_file, file_ext='.part'):
    """合并拆分的文件"""
    files = sorted([f for f in os.listdir('.') if f.startswith(input_pattern)])
    files = [f for f in files if f.endswith(file_ext)]
    with open(output_file, 'wb') as outfile:
        for filename in files:
            with open(filename, 'rb') as infile:
                outfile.write(infile.read())
            print(f"已合并: {filename}")
    print(f"完成！输出文件: {output_file}")
if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("用法: python merge_files.py <文件前缀> <输出文件>")
        sys.exit(1)
    merge_files(sys.argv[1], sys.argv[2])

注意事项

1. 二进制文件使用 'rb'/'wb' 模式
2. 大文件建议使用缓冲区读取
3. 内存管理避免一次性读取整个文件
4. 错误处理添加异常检测
5. 文件命名使用有意义的模式方便排序
6. 磁盘空间确保有足够空间存放拆分后的文件

选择哪种方法取决于你的具体需求：

• 快速处理：用 split 命令
• 需要控制：用 Python 脚本
• 复杂逻辑：用完整的 split_tool.py