Java 文件多線程下載

jopen 9年前發布 | 6K 次閱讀 Java

最近在做文件下載這塊的東西，研究了一下多線程文件下載這塊的知識。這里只說一下原理，具體實現請看代碼，已經寫了注釋了。

主要原理

為了加快下載速度，每個文件固定N個線程來下載，然后每個線程負責下載該文件的某一部分，比如文件大小90M，用3個線程來下載，那么第一個線程負責下載文件的長度范圍：0-30*1024*1024-1，第二個線程負責下載文件的長度范圍：30*1024*1024-60*1024*1024-1，第三個線程負責下載文件的長度范圍：60*1024*1024-90*1024*1024-1，3個線程下載完后就合成了整個文件。這里需要用到Http中的ContentLength和Range請求頭，ContentLength對應文件的總長度，Range頭用來請求文件某一子塊的內容，例如：Range 0-10000，表示請求該文件0-10000字節的內容。

具體代碼如下：

import java.io.File;

import java.io.IOException;

import java.io.InputStream;

import java.io.RandomAccessFile;

import java.util.zip.GZIPInputStream;
import org.apache.http.Header;

import org.apache.http.HttpResponse;

import org.apache.http.HttpStatus;

import org.apache.http.client.ClientProtocolException;

import org.apache.http.client.HttpClient;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.params.CookiePolicy;

import org.apache.http.params.CoreConnectionPNames;
import com.ricky.java.common.download.file.http.HttpClientManager;

import com.ricky.java.common.download.file.util.Constants;
public class Downloader {

private String url; // 目標地址  
private File file;  // 本地文件  
private static final int THREAD_AMOUNT = 3;                 // 線程數  
private static final String DIR_PATH = "D:/download/file";      // 下載目錄  
private long threadLen;                                     // 每個線程下載多少  

private HttpClient mHttpClient = HttpClientManager.getHttpClient();  

public Downloader(String address) throws IOException {      // 通過構造函數傳入下載地址  
    url = address;  
    file = new File(DIR_PATH, address.substring(address.lastIndexOf("?") + 1)+".xml");  
}  

public void download() throws IOException {  

    long totalLen = getContentLength(url);                          // 獲取文件長度  
    threadLen = (totalLen + THREAD_AMOUNT - 1) / THREAD_AMOUNT;         // 計算每個線程要下載的長度  

    System.out.println("totalLen="+totalLen+"***threadLen="+threadLen);  

    RandomAccessFile raf = new RandomAccessFile(file, "rws");           // 在本地創建一個和服務端大小相同的文件  
    raf.setLength(totalLen);                                            // 設置文件的大小  
    raf.close();  

    for (int i = 0; i < THREAD_AMOUNT; i++)                              // 開啟3條線程, 每個線程下載一部分數據到本地文件中  
        new DownloadThread(i).start();  
}  

public long getContentLength(String address) {  
    HttpGet httpget = null;  
    try {  
        httpget = new HttpGet(address);  
        httpget.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1");  
        httpget.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");  

        httpget.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,Constants.SO_TIMEOUT);  
        httpget.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
        httpget.getParams().setParameter("http.protocol.cookie-policy",CookiePolicy.BROWSER_COMPATIBILITY);  

        HttpResponse response = mHttpClient.execute(httpget);  

        int status = response.getStatusLine().getStatusCode();  

        if (status == HttpStatus.SC_OK) {  

            return response.getEntity().getContentLength();  
        }  
    } catch (ClientProtocolException e) {  
        e.printStackTrace();  
    } catch (IOException e) {  
        e.printStackTrace();  
    }finally{  
        if(httpget!=null){  
            httpget.abort();  
        }  
    }  
    return 0;  
}  

private class DownloadThread extends Thread {  
    private int id;   
    public DownloadThread(int id) {  
        this.id = id;  
    }  
    public void run() {  
        long start = id * threadLen;                        // 起始位置  
        long end = id * threadLen + threadLen - 1;      // 結束位置  
        System.out.println("線程" + id + ": " + start + "-" + end);  

        HttpGet httpget = null;  
        try {  
            httpget = new HttpGet(url);  
            httpget.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1");  
            httpget.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");  

            httpget.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,Constants.SO_TIMEOUT);  
            httpget.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
            httpget.getParams().setParameter("http.protocol.cookie-policy",CookiePolicy.BROWSER_COMPATIBILITY);  

            HttpResponse response = mHttpClient.execute(httpget);  

            int status = response.getStatusLine().getStatusCode();  

            if (status == HttpStatus.SC_OK) {  
                InputStream in = response.getEntity().getContent();  
                Header contentEncoding = response.getFirstHeader("Content-Encoding");  
                if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase("gzip")) {  
                    System.out.println("gzip InputStream in post");  
                    in = new GZIPInputStream(in);  
                }  

                RandomAccessFile raf = new RandomAccessFile(file, "rws");  
                raf.seek(start);  

                byte[] buffer = new byte[1024];  
                int len;  
                while ((len = in.read(buffer)) != -1)  
                    raf.write(buffer, 0, len);  
                raf.close();  

                System.out.println("線程" + id + "下載完畢");  
            }else{  
                System.out.println("線程" + id + "請求失敗,響應碼="+status);  
            }  

        } catch (ClientProtocolException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }finally{  
            if(httpget!=null){  
                httpget.abort();  
            }  
        }  
    }  
}  

public static void main(String[] args) throws IOException {  

//      new Downloader("http://dldir1.qq.com/qqfile/qq/QQ6.2/12179/QQ6.2.exe").download();

        new Downloader("http://api.t.dianping.com/n/api.xml?cityId=1").download();

    }

}  </pre> 


 HttpClientManager.java 



import org.apache.http.client.HttpClient;

import org.apache.http.conn.scheme.PlainSocketFactory;

import org.apache.http.conn.scheme.Scheme;

import org.apache.http.conn.scheme.SchemeRegistry;

import org.apache.http.conn.ssl.SSLSocketFactory;

import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.http.impl.conn.PoolingClientConnectionManager;

import org.apache.http.params.BasicHttpParams;

import org.apache.http.params.CoreConnectionPNames;

import org.apache.http.params.HttpParams;
import com.ricky.java.common.download.file.util.Constants;
public class HttpClientManager {

private static HttpParams httpParams;  
private static PoolingClientConnectionManager cm;  

/** 
 * 最大連接數 
 */  
public final static int MAX_TOTAL_CONNECTIONS = 200;  
/** 
 * 每個路由最大連接數 
 */  
public final static int MAX_ROUTE_CONNECTIONS = 300;  


static {  
    SchemeRegistry schemeRegistry = new SchemeRegistry();  
    schemeRegistry.register(  
            new Scheme("http",80,PlainSocketFactory.getSocketFactory()));  
    schemeRegistry.register(  
            new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));  

    cm = new PoolingClientConnectionManager(schemeRegistry);  
    cm.setMaxTotal(MAX_TOTAL_CONNECTIONS);  
    cm.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);  

    HttpParams params = new BasicHttpParams();  
    params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
    params.setParameter(CoreConnectionPNames.SO_TIMEOUT, Constants.SO_TIMEOUT);  
}  

public static HttpClient getHttpClient() {  
    return new DefaultHttpClient(cm, httpParams);  
}  


}  </pre> 

 如果想實現文件斷點下載的話，只需要在記錄每個線程當前下載了多少長度的內容即可，可以將其持久化到文件或數據庫中保存起來，然后線程開始下載的時候都先讀取一下它當前下載了多少就OK了。

本文由用戶 jopen 自行上傳分享，僅供網友學習交流。所有權歸原作者，若您的權利被侵害，請聯系管理員。

轉載本站原創文章，請注明出處，并保留原始鏈接、圖片水印。

本站是一個以用戶分享為主的開源技術平臺，歡迎各類分享！

本文地址：http://www.baiduhome.net/code/view/1441896716742

Java

Java 文件多線程下載

相關代碼

相關文檔

相關經驗

目錄