Java集合去重案例如何实现

本文目录导读：

1. List去重（保持顺序）
2. 对象去重（自定义比较规则）
3. 根据对象某个字段去重
4. 性能优化：大数据量去重
5. 综合示例：完整的工具类方法

在Java中,集合去重是一个非常常见的操作，根据不同的集合类型和具体需求，有多种实现方式，下面是几种常见的Java集合去重案例及实现方法。

List去重（保持顺序）

使用LinkedHashSet

import java.util.*;
public class ListDeduplicationExample {
    public static void main(String[] args) {
        List<Integer> list = Arrays.asList(1, 2, 2, 3, 4, 4, 5, 1, 6);
        // 使用LinkedHashSet保持插入顺序
        Set<Integer> set = new LinkedHashSet<>(list);
        List<Integer> result = new ArrayList<>(set);
        System.out.println("原始列表: " + list);
        System.out.println("去重后列表: " + result);
    }
}

使用Stream API (Java 8+)

import java.util.*;
import java.util.stream.Collectors;
public class StreamDeduplicationExample {
    public static void main(String[] args) {
        List<String> list = Arrays.asList("apple", "banana", "apple", "orange", "banana", "grape");
        // 使用Stream API去重
        List<String> result = list.stream()
                                .distinct()
                                .collect(Collectors.toList());
        System.out.println("原始列表: " + list);
        System.out.println("去重后列表: " + result);
    }
}

对象去重（自定义比较规则）

重写equals和hashCode

import java.util.*;
import java.util.stream.Collectors;
class Person {
    private String name;
    private int age;
    public Person(String name, int age) {
        this.name = name;
        this.age = age;
    }
    // 重写equals方法
    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;
        Person person = (Person) o;
        return age == person.age && Objects.equals(name, person.name);
    }
    // 重写hashCode方法
    @Override
    public int hashCode() {
        return Objects.hash(name, age);
    }
    @Override
    public String toString() {
        return "Person{" + "name='" + name + '\'' + ", age=" + age + '}';
    }
}
public class ObjectDeduplicationExample {
    public static void main(String[] args) {
        List<Person> persons = Arrays.asList(
            new Person("Alice", 25),
            new Person("Bob", 30),
            new Person("Alice", 25),  // 重复
            new Person("Charlie", 35),
            new Person("Bob", 30)     // 重复
        );
        // 使用Set去重（依赖equals和hashCode）
        Set<Person> personSet = new LinkedHashSet<>(persons);
        List<Person> result = new ArrayList<>(personSet);
        System.out.println("原始对象列表:");
        persons.forEach(System.out::println);
        System.out.println("\n去重后对象列表:");
        result.forEach(System.out::println);
    }
}

使用TreeSet自定义比较器

import java.util.*;
public class CustomComparatorDeduplicationExample {
    public static void main(String[] args) {
        List<Product> products = Arrays.asList(
            new Product("A001", "笔记本电脑", 5000),
            new Product("A002", "手机", 3000),
            new Product("A001", "笔记本电脑", 5000),  // 重复ID
            new Product("A003", "平板", 2000)
        );
        // 使用TreeSet根据产品ID去重
        Set<Product> productSet = new TreeSet<>((p1, p2) -> p1.getId().compareTo(p2.getId()));
        productSet.addAll(products);
        System.out.println("根据不同ID去重后的产品列表:");
        productSet.forEach(System.out::println);
    }
}
class Product {
    private String id;
    private String name;
    private double price;
    public Product(String id, String name, double price) {
        this.id = id;
        this.name = name;
        this.price = price;
    }
    public String getId() { return id; }
    @Override
    public String toString() {
        return "Product{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", price=" + price + '}';
    }
}

根据对象某个字段去重

import java.util.*;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.collectingAndThen;
import static java.util.stream.Collectors.toCollection;
public class FieldDeduplicationExample {
    public static void main(String[] args) {
        List<Student> students = Arrays.asList(
            new Student(1, "张三", "A班"),
            new Student(2, "李四", "B班"),
            new Student(1, "张三", "C班"),  // 同一个学生，不同班级
            new Student(3, "王五", "A班"),
            new Student(2, "李四", "B班")   // 重复
        );
        // 方法1：根据学生ID去重，保留第一个出现的
        List<Student> result1 = students.stream()
            .collect(Collectors.collectingAndThen(
                Collectors.toMap(Student::getId, s -> s, (existing, replacement) -> existing),
                map -> new ArrayList<>(map.values())
            ));
        // 方法2：使用TreeSet
        Set<Student> set = new TreeSet<>(Comparator.comparing(Student::getId));
        set.addAll(students);
        List<Student> result2 = new ArrayList<>(set);
        System.out.println("根据ID去重后的学生列表（方法1）:");
        result1.forEach(System.out::println);
        System.out.println("\n根据ID去重后的学生列表（方法2）:");
        result2.forEach(System.out::println);
    }
}
class Student {
    private int id;
    private String name;
    private String className;
    public Student(int id, String name, String className) {
        this.id = id;
        this.name = name;
        this.className = className;
    }
    public int getId() { return id; }
    @Override
    public String toString() {
        return "Student{" + "id=" + id + ", name='" + name + '\'' + ", className='" + className + '\'' + '}';
    }
}

性能优化：大数据量去重

import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public class LargeDataDeduplicationExample {
    public static void main(String[] args) {
        // 模拟大数据量
        List<String> largeList = new ArrayList<>();
        for (int i = 0; i < 1000000; i++) {
            largeList.add("item_" + (i % 100000));  // 约10%重复
        }
        // 方法1：使用HashSet（快速，但不保持顺序）
        long startTime = System.currentTimeMillis();
        Set<String> set = new HashSet<>(largeList);
        List<String> result1 = new ArrayList<>(set);
        long endTime = System.currentTimeMillis();
        System.out.println("HashSet去重耗时: " + (endTime - startTime) + "ms");
        System.out.println("去重后数量: " + result1.size());
        // 方法2：使用Parallel Stream（并行处理）
        startTime = System.currentTimeMillis();
        List<String> result2 = largeList.parallelStream()
                                      .distinct()
                                      .collect(Collectors.toList());
        endTime = System.currentTimeMillis();
        System.out.println("Parallel Stream去重耗时: " + (endTime - startTime) + "ms");
        System.out.println("去重后数量: " + result2.size());
    }
}

综合示例：完整的工具类方法

import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
public class DeduplicationUtil {
    // 1. List去重（保持顺序）
    public static <T> List<T> distinctList(List<T> list) {
        if (list == null || list.isEmpty()) {
            return new ArrayList<>();
        }
        return new ArrayList<>(new LinkedHashSet<>(list));
    }
    // 2. 根据对象字段去重
    public static <T, R> List<T> distinctByField(List<T> list, Function<? super T, ? extends R> keyExtractor) {
        if (list == null || list.isEmpty()) {
            return new ArrayList<>();
        }
        return list.stream()
            .filter(distinctByKey(keyExtractor))
            .collect(Collectors.toList());
    }
    // 辅助方法：根据key去重
    private static <T, R> Predicate<T> distinctByKey(Function<? super T, ? extends R> keyExtractor) {
        Set<R> seen = ConcurrentHashMap.newKeySet();
        return t -> seen.add(keyExtractor.apply(t));
    }
    // 3. 合并并去重两个列表
    public static <T> List<T> mergeAndDistinct(List<T> list1, List<T> list2) {
        Set<T> set = new LinkedHashSet<>();
        if (list1 != null) set.addAll(list1);
        if (list2 != null) set.addAll(list2);
        return new ArrayList<>(set);
    }
    public static void main(String[] args) {
        // 测试工具类
        List<Integer> list1 = Arrays.asList(1, 2, 3, 4, 2, 3);
        List<Integer> list2 = Arrays.asList(3, 4, 5, 6, 1);
        System.out.println("原始列表1: " + list1);
        System.out.println("原始列表2: " + list2);
        System.out.println("去重后列表1: " + distinctList(list1));
        System.out.println("合并去重后: " + mergeAndDistinct(list1, list2));
    }
}

选择哪种方法取决于你的具体需求：是否需要保持顺序、数据量大小、是否需要自定义比较规则等。