java超快速文本去重復代碼

import java.io.*;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

public class SpeedClear {

public static void main(String[] args) {
    if(args.length==0){
        print();
        System.exit(1);
    }
    if(args.length!=2){
        System.out.println("Format error...");
        System.exit(1);
    }
    String pathname = args[0];
    String newPath = args[1];
    clear(pathname,newPath);     //調用去重復的方法...
}

/**
 * 
 * @param pathname
 *            源文件路徑
 * @param newPath
 *            新的文件路徑
 * @throws Exception
 */
public static void clear(String pathname, String newPath) {

    System.out.println("Start... ");

    try{    //懶的寫Try..直接都包圍起來吧....

        File file = new File(pathname);
    BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file));


BufferedReader buffer = new BufferedReader(new InputStreamReader(fis,"utf-8"),2010241024);// 用5M的緩沖讀取文本文件

        //FileWriter fw  = new FileWriter(new File(newPath),true);  //去除后的文本

        OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(new File(newPath)),"utf-8") ;

        Set<String> set = new HashSet<String>();
        String temp = ""; // 臨時字符串
        int x = 0;
        while ((temp = buffer.readLine()) != null) { // 讀文件,一行讀一個
            set.add(temp); // 存儲到Set集合里面
            if(x%30000==0){
                System.out.print("..") ;
            }
            x++;
        }
        fis.close();
        buffer.close();   //關閉讀取操作

        //下面開始寫文件

        for (String xxser : set) {
            out.write(xxser+"\r\n");

        }
        System.out.println("") ;
        out.close();   //關閉寫操作
        System.out.println("size = " + set.size());
        System.out.println("End...");
    }catch(Exception e){

        System.out.println("文件太大了,建議先100MB大小..") ;
    }


}


    public static void  print(){
    System.out.println("*************************************************");
    System.out.println("\t\tTo repeat \t\t");
    System.out.println();
    System.out.println("  format: java -Xmx1000m SpeedClear c:\\old.txt c:\\new.txt\t\t");
    System.out.println();
    System.out.println("\t\tAuthor:xxser   QQ:616100108");
    System.out.println("*************************************************");

}


}
 </pre>