本文目录导读：
基础实现方案
使用Python的文本摘要库
# 方法1：使用sumy库
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lsa import LsaSummarizer
def generate_summary(text, sentences_count=3):
parser = PlaintextParser.from_string(text, Tokenizer("english"))
summarizer = LsaSummarizer()
summary = summarizer(parser.document, sentences_count)
return " ".join([str(sentence) for sentence in summary])
# 方法2：使用transformers（推荐）
from transformers import pipeline
def generate_abstractive_summary(text, max_length=150):
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
summary = summarizer(text, max_length=max_length, min_length=30)
return summary[0]['summary_text']
完整实现脚本
import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from collections import Counter
import numpy as np
class TextSummarizer:
def __init__(self):
nltk.download('punkt', quiet=True)
nltk.download('stopwords', quiet=True)
self.stop_words = set(stopwords.words('english'))
def preprocess_text(self, text):
"""文本预处理"""
# 去除特殊字符
text = re.sub(r'[^\w\s]', '', text)
text = re.sub(r'\s+', ' ', text).strip()
return text.lower()
def extractive_summarization(self, text, num_sentences=5):
"""抽取式摘要"""
# 分句
sentences = sent_tokenize(text)
# 计算词频
words = word_tokenize(self.preprocess_text(text))
word_freq = Counter([word for word in words if word not in self.stop_words])
# 计算句子得分
sentence_scores = {}
for sentence in sentences:
sentence_words = word_tokenize(sentence.lower())
score = sum([word_freq[word] for word in sentence_words if word in word_freq])
sentence_scores[sentence] = score
# 选择得分最高的句子
top_sentences = sorted(sentence_scores.items(), key=lambda x: x[1], reverse=True)[:num_sentences]
summary = ' '.join([sent[0] for sent in top_sentences])
return summary
def abstractive_summarization(self, text, ratio=0.3):
"""生成式摘要（基于TF-IDF）"""
from sklearn.feature_extraction.text import TfidfVectorizer
sentences = sent_tokenize(text)
# 计算TF-IDF
vectorizer = TfidfVectorizer(stop_words='english')
tfidf_matrix = vectorizer.fit_transform(sentences)
# 计算句子重要性
sentence_scores = np.array(tfidf_matrix.sum(axis=1)).flatten()
# 选择重要句子
num_sentences = max(1, int(len(sentences) * ratio))
top_indices = sentence_scores.argsort()[-num_sentences:][::-1]
top_indices.sort()
summary = ' '.join([sentences[i] for i in top_indices])
return summary
# 使用示例
summarizer = TextSummarizer()
long_text = """
Artificial intelligence (AI) is transforming the way we live and work.
From virtual assistants like Siri and Alexa to self-driving cars, AI technologies
have become an integral part of our daily lives. Machine learning, a subset of AI,
allows computers to learn from data without being explicitly programmed.
Deep learning, a further subset, uses neural networks with multiple layers to analyze
various factors of data. These technologies are being applied in healthcare for
diagnosis, in finance for fraud detection, and in manufacturing for quality control.
The future of AI holds even more promise, with potential applications in climate
change mitigation, space exploration, and solving complex global challenges.
However, concerns about job displacement, privacy, and ethical use of AI continue
to be debated by experts and policymakers worldwide.
"""
extractive_summary = summarizer.extractive_summarization(long_text, num_sentences=3)
print("Extractive Summary:")
print(extractive_summary)
abstractive_summary = summarizer.abstractive_summarization(long_text, ratio=0.4)
print("\nAbstractive Summary:")
print(abstractive_summary)
高级实现 - 基于LLM的摘要生成
import openai
from typing import List, Dict
class LLMSummarizer:
def __init__(self, api_key: str):
openai.api_key = api_key
def chunk_text(self, text: str, max_tokens: int = 3000) -> List[str]:
"""分割长文本为块"""
sentences = sent_tokenize(text)
chunks = []
current_chunk = ""
for sentence in sentences:
if len(current_chunk) + len(sentence) < max_tokens:
current_chunk += sentence + " "
else:
chunks.append(current_chunk.strip())
current_chunk = sentence + " "
if current_chunk:
chunks.append(current_chunk.strip())
return chunks
def generate_summary_with_llm(self, text: str, summary_length: str = "medium") -> str:
"""使用LLM生成摘要"""
chunks = self.chunk_text(text)
summaries = []
for chunk in chunks:
prompt = f"""请为以下文本生成一个简洁的摘要（{summary_length}长度）：
文本：{chunk}
保持关键信息，语言流畅）："""
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "你是一个专业的文本摘要生成器。"},
{"role": "user", "content": prompt}
],
max_tokens=300 if summary_length == "short" else 500,
temperature=0.5
)
summaries.append(response.choices[0].message.content)
# 如果有多个摘要块，合并它们
if len(summaries) > 1:
combined_prompt = f"""请将以下多个摘要合并为一个连贯的完整摘要：
{' '.join(summaries)}
请确保合并后的摘要流畅自然，保留所有重要信息："""
final_response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": combined_prompt}
],
max_tokens=500,
temperature=0.3
)
return final_response.choices[0].message.content
return summaries[0]
# 使用示例
# summarizer = LLMSummarizer(api_key="your_openai_api_key")
# summary = summarizer.generate_summary_with_llm(long_text, summary_length="medium")
# print(summary)
基于规则的摘要生成
def rule_based_summarization(text: str, max_sentences: int = 5) -> str:
"""基于规则的摘要生成"""
# 1. 句子分割
sentences = sent_tokenize(text)
# 2. 提取特征词
important_words = set()
# 标题词（如果有标题则优先）word_patterns = [
r'\b[A-Z][a-z]+ [A-Z][a-z]+\b', # 专有名词
r'\b\d+[%]\b', # 数字+百分比
r'\b(new|important|significant|key|major)\b', # 强调词
r'\b(conclusion|result|finding|discovery)\b' # 总结词
]
for pattern in title_word_patterns:
matches = re.findall(pattern, text, re.IGNORECASE)
important_words.update(matches)
# 3. 计算句子权重
sentence_weights = {}
for i, sentence in enumerate(sentences):
weight = 0
# 位置权重（开头的句子更重要）
position_weight = 1.0 / (i + 1)
weight += position_weight * 2
# 重要词权重
sentence_words = word_tokenize(sentence.lower())
for word in sentence_words:
if word.lower() in important_words:
weight += 0.5
elif len(word) > 6: # 长词通常更重要
weight += 0.3
# 长度权重（过长或过短的句子降低权重）
if len(sentence_words) < 5 or len(sentence_words) > 50:
weight *= 0.5
sentence_weights[sentence] = weight
# 4. 选择最高权重的句子
top_sentences = sorted(
sentence_weights.items(),
key=lambda x: x[1],
reverse=True
)[:max_sentences]
# 5. 按原文顺序重组
result_sentences = []
for sentence in sentences:
for top_sentence, _ in top_sentences:
if sentence == top_sentence:
result_sentences.append(sentence)
break
return ' '.join(result_sentences)
# 使用示例
rule_based_summary = rule_based_summarization(long_text, max_sentences=3)
print("Rule-based Summary:")
print(rule_based_summary)
完整的Web服务脚本
from flask import Flask, request, jsonify
import json
app = Flask(__name__)
class SummaryService:
def __init__(self):
self.summarizer = TextSummarizer()
def generate_summary(self, text: str, method: str = "extractive",
sentences_count: int = 3) -> Dict:
"""生成摘要的Web服务方法"""
try:
if not text or len(text.strip()) < 50:
return {"error": "Text too short for summarization", "status": 400}
if method == "extractive":
summary = self.summarizer.extractive_summarization(text, sentences_count)
elif method == "abstractive":
summary = self.summarizer.abstractive_summarization(text, 0.3)
else:
return {"error": "Invalid method", "status": 400}
return {
"summary": summary,
"original_length": len(text),
"summary_length": len(summary),
"compression_ratio": f"{len(summary)/len(text)*100:.1f}%",
"method": method,
"status": 200
}
except Exception as e:
return {"error": str(e), "status": 500}
summary_service = SummaryService()
@app.route('/api/summarize', methods=['POST'])
def summarize():
data = request.get_json()
text = data.get('text', '')
method = data.get('method', 'extractive')
sentences_count = data.get('sentences_count', 3)
result = summary_service.generate_summary(text, method, sentences_count)
if result.get('status') == 200:
return jsonify(result), 200
else:
return jsonify(result), result.get('status', 500)
# 启动服务
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)
常用的文本摘要库
安装依赖
# 基础库 pip install nltk sumy scikit-learn # 深度学习库 pip install transformers torch # 文本处理 pip install textblob pattern # 服务框架 pip install flask
优化建议
性能优化
import hashlib
from functools import lru_cache
@lru_cache(maxsize=100)
def cached_summarization(text_hash: str, method: str):
"""缓存摘要结果"""
# 实际的摘要逻辑
pass
def get_text_hash(text: str) -> str:
"""获取文本哈希值，用于缓存"""
return hashlib.md5(text.encode()).hexdigest()
质量提升技巧
- 使用多种方法的组合
- 添加关键词提取增强
- 实现多语言支持
- 考虑文本类型适配
这个脚本可以灵活扩展,根据你的具体需求选择合适的实现方案，如果需要处理特定类型的文本，可以进一步定制规则和模型。
