本文目录导读：
我来为你介绍几种清理无效链接的脚本方案,根据不同的使用场景选择适合的方法：
网页HTML中的无效链接检查
Python脚本（使用requests + BeautifulSoup）
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import concurrent.futures
def check_link(url):
"""检查单个链接是否有效"""
try:
response = requests.head(url, timeout=5, allow_redirects=True)
if response.status_code == 404:
return (url, False, "404 Not Found")
elif response.status_code >= 400:
return (url, False, f"HTTP {response.status_code}")
return (url, True, "OK")
except requests.exceptions.SSLError:
return (url, False, "SSL Error")
except requests.exceptions.ConnectionError:
return (url, False, "Connection Error")
except requests.exceptions.Timeout:
return (url, False, "Timeout")
except Exception as e:
return (url, False, str(e))
def find_links(page_url):
"""获取页面中所有链接"""
try:
response = requests.get(page_url, timeout=10)
soup = BeautifulSoup(response.text, 'html.parser')
links = []
for link in soup.find_all('a'):
href = link.get('href')
if href and not href.startswith(('#', 'mailto:', 'tel:', 'javascript:')):
absolute_url = urljoin(page_url, href)
links.append(absolute_url)
return links
except Exception as e:
print(f"Error fetching {page_url}: {e}")
return []
def clean_invalid_links(page_url):
"""主函数：检查并清理无效链接"""
print(f"正在分析页面: {page_url}")
# 获取所有链接
all_links = find_links(page_url)
print(f"找到 {len(all_links)} 个链接")
# 使用线程池并发检查链接
invalid_links = []
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
future_to_link = {executor.submit(check_link, link): link for link in all_links}
for future in concurrent.futures.as_completed(future_to_link):
link = future_to_link[future]
try:
url, is_valid, message = future.result()
if not is_valid:
invalid_links.append((url, message))
print(f"❌ 无效: {url} - {message}")
else:
print(f"✅ 有效: {url}")
except Exception as e:
print(f"Error checking {link}: {e}")
# 输出结果
print(f"\n===== 检查完成 =====")
print(f"无效链接数量: {len(invalid_links)}")
# 保存到文件
with open('invalid_links.txt', 'w') as f:
for url, error in invalid_links:
f.write(f"{url} || {error}\n")
return invalid_links
if __name__ == "__main__":
# 使用示例
target_url = "http://example.com" # 替换为你的目标URL
clean_invalid_links(target_url)
Linux/Unix系统脚本（Bash）
清理文件系统中的无效符号链接
#!/bin/bash
# 清理无效符号链接
find_valid_links() {
local dir="$1"
echo "开始检查 $dir 下的符号链接..."
# 查找所有符号链接
find "$dir" -type l 2>/dev/null | while read -r link; do
# 检查目标是否存在
if [ ! -e "$link" ]; then
echo "❌ 无效链接: $link"
# 可选：自动删除（取消下面注释启用）
# rm "$link"
else
echo "✅ 有效链接: $link"
fi
done
}
# 清理HP/Unix系统中的无效链接
clean_invalid_links() {
local target_dir="${1:-.}"
echo "开始扫描目录: $target_dir"
# 查找并删除无效链接
find "$target_dir" -type l -exec test ! -e {} \; -delete -print 2>/dev/null | while read -r file; do
echo "已删除无效链接: $file"
done
echo "清理完成"
}
# 简化版：检查并清理
main() {
echo "清理无效链接脚本"
echo "================="
# 选择模式
case "${1:-check}" in
check)
find_valid_links "${2:-.}"
;;
clean)
clean_invalid_links "${2:-.}"
;;
*)
echo "用法: $0 [check|clean] [目录]"
exit 1
;;
esac
}
main "$@"
数据库链接检查（MySQL）
-- 清理数据库中的无效链接
DELIMITER $$
CREATE PROCEDURE CleanInvalidLinks()
BEGIN
DECLARE done INT DEFAULT FALSE;
DECLARE link_url VARCHAR(255);
DECLARE cur CURSOR FOR SELECT url FROM your_table;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
OPEN cur;
read_loop: LOOP
FETCH cur INTO link_url;
IF done THEN
LEAVE read_loop;
END IF;
-- 模拟检查（实际需要使用外部工具）
-- 这里简单示例：检查URL格式
IF link_url NOT LIKE 'http%://%' THEN
DELETE FROM your_table WHERE url = link_url;
SELECT CONCAT('已删除无效链接: ', link_url) AS result;
END IF;
END LOOP;
CLOSE cur;
END$$
DELIMITER ;
Node.js脚本（用于爬虫/网页）
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
async function checkLink(url) {
try {
const response = await axios.head(url, { timeout: 5000 });
return { url, valid: response.status < 400 };
} catch (error) {
return { url, valid: false, error: error.message };
}
}
async function cleanLinks(pageUrl) {
console.log(`开始检查页面: ${pageUrl}`);
try {
const response = await axios.get(pageUrl);
const $ = cheerio.load(response.data);
const links = [];
// 获取所有链接
$('a').each((index, element) => {
const href = $(element).attr('href');
if (href && !href.startsWith('#')) {
links.push(new URL(href, pageUrl).href);
}
});
console.log(`共找到 ${links.length} 个链接`);
// 并发检查链接
const results = await Promise.all(links.map(checkLink));
const invalidLinks = results.filter(result => !result.valid);
console.log(`无效链接: ${invalidLinks.length} 个`);
// 保存结果
fs.writeFileSync('invalid_links.txt',
invalidLinks.map(l => `${l.url} - ${l.error || 'Invalid'}`).join('\n')
);
return invalidLinks;
} catch (error) {
console.error('获取页面失败:', error.message);
return [];
}
}
// 使用示例
cleanLinks('http://example.com');
正则表达式批量清理
import re
中清理无效链接
def clean_html_links(file_path):
with open(file_path, 'r', encoding='utf-8') as file:
content = file.read()
# 匹配并检查链接
url_pattern = r'https?://[^\s<>"\'{}|\\^`\[\]]+'
urls = re.findall(url_pattern, content)
invalid_count = 0
for url in urls:
# 简单有效性检查
if url.count('.') < 2: # 简单的域名检查
content = content.replace(url, '')
invalid_count += 1
print(f"已移除无效链接: {url}")
# 保存清理后的文件
with open(file_path, 'w', encoding='utf-8') as file:
file.write(content)
print(f"共清理了 {invalid_count} 个无效链接")
使用方法
- 网页链接检查：运行Python脚本，指定要检查的页面URL
- 系统链接检查：运行Bash脚本，可以检查当前目录或指定目录
- 数据库清理：执行MySQL存储过程
- 爬虫项目：使用Node.js脚本集成到爬虫中
需要根据实际需求选择合适的方案，如果需要更具体的应用场景，可以告诉我你的具体需求,我帮你调整。
