本文目录导读：
我来介绍几种拆分大文件的方法,包括命令行和Python脚本。
Linux/Mac 命令行split工具
按大小拆分
# 每个文件100MB split -b 100M large_file.txt part_ # 每个文件100MB，使用数字后缀 split -b 100M -d large_file.txt part_ # 每个文件100MB，数字后缀，指定后缀长度 split -b 100m -d -a 3 large_file.txt part_
按行数拆分
# 每10000行一个文件 split -l 10000 large_file.txt part_ # 每10000行，数字后缀 split -l 10000 -d large_file.txt part_
Python脚本拆分
按文件大小拆分
#!/usr/bin/env python3
# split_by_size.py
import os
def split_file_by_size(input_file, output_prefix, chunk_size_mb):
"""按指定大小拆分文件"""
chunk_size = chunk_size_mb * 1024 * 1024 # 转换为字节
file_count = 0
bytes_written = 0
with open(input_file, 'rb') as infile:
while True:
chunk = infile.read(chunk_size)
if not chunk:
break
output_file = f"{output_prefix}_{file_count:03d}.part"
with open(output_file, 'wb') as outfile:
outfile.write(chunk)
file_count += 1
bytes_written += len(chunk)
print(f"已创建: {output_file} ({len(chunk)} bytes)")
print(f"\n完成！共拆分 {file_count} 个文件，总大小 {bytes_written / 1024 / 1024:.2f} MB")
if __name__ == "__main__":
import sys
if len(sys.argv) != 4:
print("用法: python split_by_size.py <输入文件> <输出前缀> <大小MB>")
sys.exit(1)
split_file_by_size(sys.argv[1], sys.argv[2], int(sys.argv[3]))
按行数拆分
#!/usr/bin/env python3
# split_by_lines.py
def split_file_by_lines(input_file, output_prefix, lines_per_file):
"""按行数拆分文件"""
file_count = 0
line_count = 0
current_file = None
with open(input_file, 'r', encoding='utf-8') as infile:
for line in infile:
if line_count % lines_per_file == 0:
if current_file:
current_file.close()
output_file = f"{output_prefix}_{file_count:03d}.txt"
current_file = open(output_file, 'w', encoding='utf-8')
file_count += 1
print(f"正在写入: {output_file}")
current_file.write(line)
line_count += 1
if current_file:
current_file.close()
print(f"\n完成！共拆分 {file_count} 个文件，总行数 {line_count}")
if __name__ == "__main__":
import sys
if len(sys.argv) != 4:
print("用法: python split_by_lines.py <输入文件> <输出前缀> <行数>")
sys.exit(1)
split_file_by_lines(sys.argv[1], sys.argv[2], int(sys.argv[3]))
通用拆分工具（支持二进制和文本）
#!/usr/bin/env python3
# split_tool.py
import os
import argparse
from pathlib import Path
class FileSplitter:
def __init__(self, input_file, output_dir=None, buffer_size=1024*1024):
self.input_file = Path(input_file)
self.output_dir = output_dir or self.input_file.parent
self.buffer_size = buffer_size
self.file_size = self.input_file.stat().st_size
os.makedirs(self.output_dir, exist_ok=True)
def split_by_size(self, chunk_size_mb):
"""按大小拆分"""
chunk_size = chunk_size_mb * 1024 * 1024
parts = 0
with open(self.input_file, 'rb') as infile:
while True:
chunk = infile.read(chunk_size)
if not chunk:
break
output_file = self.output_dir / f"{self.input_file.stem}_part{parts:03d}{self.input_file.suffix}"
with open(output_file, 'wb') as outfile:
outfile.write(chunk)
parts += 1
print(f"拆分第 {parts} 部分: {output_file}")
return parts
def split_by_lines(self, lines_per_file):
"""按行拆分（文本文件）"""
parts = 0
line_count = 0
current_file = None
with open(self.input_file, 'r', encoding='utf-8') as infile:
for line in infile:
if line_count % lines_per_file == 0:
if current_file:
current_file.close()
output_file = self.output_dir / f"{self.input_file.stem}_part{parts:03d}.txt"
current_file = open(output_file, 'w', encoding='utf-8')
parts += 1
current_file.write(line)
line_count += 1
if current_file:
current_file.close()
return parts
def split_by_count(self, num_parts):
"""按数量均分"""
chunk_size = self.file_size // num_parts
parts = 0
with open(self.input_file, 'rb') as infile:
for i in range(num_parts):
if i == num_parts - 1: # 最后一个文件包含剩余所有内容
chunk = infile.read()
else:
chunk = infile.read(chunk_size)
if not chunk:
break
output_file = self.output_dir / f"{self.input_file.stem}_part{i:03d}{self.input_file.suffix}"
with open(output_file, 'wb') as outfile:
outfile.write(chunk)
parts += 1
return parts
# 命令行接口
def main():
parser = argparse.ArgumentParser(description='拆分大文件工具')
parser.add_argument('input', help='输入文件路径')
parser.add_argument('-o', '--output-dir', help='输出目录（默认：输入文件所在目录）')
parser.add_argument('-s', '--size', type=float, help='按大小拆分（MB）')
parser.add_argument('-l', '--lines', type=int, help='按行数拆分（文本文件）')
parser.add_argument('-n', '--num-parts', type=int, help='按数量均分')
args = parser.parse_args()
splitter = FileSplitter(args.input, args.output_dir)
if args.size:
parts = splitter.split_by_size(args.size)
print(f"按大小拆分完成，共 {parts} 个文件")
elif args.lines:
parts = splitter.split_by_lines(args.lines)
print(f"按行数拆分完成，共 {parts} 个文件")
elif args.num_parts:
parts = splitter.split_by_count(args.num_parts)
print(f"按数量拆分完成，共 {parts} 个文件")
else:
parser.print_help()
if __name__ == "__main__":
main()
使用示例
命令行使用
# Linux/Mac split -b 100m large_file.zip part_ split -l 10000 data.csv data_part_ # Python脚本使用 python split_tool.py large_file.zip -s 100 -o output_dir python split_tool.py data.csv -l 10000 -o output_dir python split_tool.py video.mp4 -n 5 -o output_dir
合并拆分的文件
#!/usr/bin/env python3
# merge_files.py
import os
import sys
def merge_files(input_pattern, output_file, file_ext='.part'):
"""合并拆分的文件"""
files = sorted([f for f in os.listdir('.') if f.startswith(input_pattern)])
files = [f for f in files if f.endswith(file_ext)]
with open(output_file, 'wb') as outfile:
for filename in files:
with open(filename, 'rb') as infile:
outfile.write(infile.read())
print(f"已合并: {filename}")
print(f"完成！输出文件: {output_file}")
if __name__ == "__main__":
if len(sys.argv) != 3:
print("用法: python merge_files.py <文件前缀> <输出文件>")
sys.exit(1)
merge_files(sys.argv[1], sys.argv[2])
注意事项
- 二进制文件使用
'rb'/'wb'模式 - 大文件建议使用缓冲区读取
- 内存管理避免一次性读取整个文件
- 错误处理添加异常检测
- 文件命名使用有意义的模式方便排序
- 磁盘空间确保有足够空间存放拆分后的文件
选择哪种方法取决于你的具体需求：
- 快速处理：用
split命令 - 需要控制：用 Python 脚本
- 复杂逻辑：用完整的 split_tool.py
