Java多线程下载文件

Java多线程下载文件

Java多线程下载文件

优化：合理利用服务器资源，将资源利用最大化，加快下载速度

一般有两种方式：

多线程如果只知道和，那你就Out了，Java1.5以后有个包，就是Java并发编程。

N个线程下载单个文件

定义一个单个线程下载的部分

public class DownloadWithRange implements Runnable {
    private String urlLocation;
    private String filePath;
    private long start;
    private long end;
    DownloadWithRange(String urlLocation, String filePath, long start, long end) {
        this.urlLocation = urlLocation;
        this.filePath = filePath;
        this.start = start;
        this.end = end;
    }
    @Override
    public void run() {
        try {
            HttpURLConnection conn = getHttp();
            conn.setRequestProperty("Range", "bytes=" + start + "-" + end);
            File file = new File(filePath);
            RandomAccessFile out = null;
            if (file != null) {
                out = new RandomAccessFile(file, "rw");
            }
            out.seek(start);
            InputStream in = conn.getInputStream();
            byte[] b = new byte[1024];
            int len = 0;
            while ((len = in.read(b)) >= 0) {
                out.write(b, 0, len);
            }
            in.close();
            out.close();
        } catch (Exception e) {
            e.getMessage();
        }
    }
    public HttpURLConnection getHttp() throws IOException {
        URL url = null;
        if (urlLocation != null) {
            url = new URL(urlLocation);
        }
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setReadTimeout(5000);
        conn.setRequestMethod("GET");
        return conn;
    }
}

定义线程Pool

public class DownloadFileWithThreadPool {
    public void getFileWithThreadPool(String urlLocation, String filePath, int poolLength) throws IOException {
        ExecutorService threadPool = Executors.newCachedThreadPool();
        long len = getContentLength(urlLocation);
        System.out.println(len);
        for (int i = 0; i < poolLength; i++) {
            long start = i * len / poolLength;
            long end = (i + 1) * len / poolLength - 1;
            if (i == poolLength - 1) {
                end = len;
            }
            System.out.println(start+"---------------"+end);
            DownloadWithRange download = new DownloadWithRange(urlLocation, filePath, start, end);
            threadPool.execute(download);
        }
        threadPool.shutdown();
    }
    public static long getContentLength(String urlLocation) throws IOException {
        URL url = null;
        if (urlLocation != null) {
            url = new URL(urlLocation);
        }
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setReadTimeout(5000);
        conn.setRequestMethod("GET");
        long len = conn.getContentLength();
        return len;
    }
}

测试下载单个文件

public static void main(String[] args) {
        Date startDate = new Date();
        DownloadFileWithThreadPool pool = new DownloadFileWithThreadPool();
        try {
            pool.getFileWithThreadPool("http://mpge.5nd.com/2016/2016-11-15/74847/1.mp3", "D:\1.mp3", 100);
        } catch (IOException e) {
            e.printStackTrace();
        }
        System.out.println(new Date().getTime() - startDate.getTime());
    }

测试发现，我目前的网速java并发写文件，下载一个3.6M左右mp3音频文件，使用多线程，下载大概需要800ms左右，而不使用多线程，则需要以上，有时候甚至

多文件下载，单文件单线程

/**
 * @author Nick 带线程池的文件下载类，线程大小10
 *      文件数少的情况下，体现不大
 * @version V1.0.0
 * @Date 2017/8/2 20:43
 */
public class FileDownConnManager {
    private static final Logger logger = LoggerFactory.getLogger(FileDownConnManager.class);
    private static final FileDownConnManager connManager = new FileDownConnManager();
    private static ExecutorService executorService = Executors.newFixedThreadPool(10); //10个线程跑
    public static FileDownConnManager getDefaultManager() {
        return connManager;
    }
    public static byte[] fileDown(final String netURL) throws ExecutionException, InterruptedException {
        Future<byte[]> future = executorService.submit(new Callable<byte[]>() {
            @Override
            public byte[] call() throws Exception {
                Date date = new Date();
                URL url;
                byte[] getData = new byte[0];
                InputStream is = null;
                try {
                    url = new URL(netURL);
                    URLConnection connection = url.openConnection();
                    is = connection.getInputStream();
                    getData = readInputStream(is);
                } catch (IOException e) {
                    logger.error("从URL获得字节流数组失败 " + ExceptionUtils.getMessage(e));
                } finally {
                    try {
                        is.close();
                    } catch (IOException e) {
                        logger.error("从URL获得字节流数组流释放失败 " + ExceptionUtils.getMessage(e));
                    }
                }
                return getData;
            }
        });
        return future.get();
    }
}

测试

@Test
    public void test2() throws ExecutionException, InterruptedException, IOException {
        long time1 = System.currentTimeMillis();
        for(int i = 0; i < 15; i++) {
            byte[] by1 = FileDownConnManager.fileDown("http://mpge.5nd.com/2016/2016-11-15/74847/1.mp3");
            FileUtils.writeByteArrayToFile(new File("D:\test2_"+i+".mp3"), by1);
        }
        System.out.println(System.currentTimeMillis() - time1);
    }
    @Test
    public void test3() throws IOException {
        long time1 = System.currentTimeMillis();
        for(int i = 0; i < 15; i++) {
            byte[] by1 = FileFromUrlUtil.getInputStreamFromUrl("http://mpge.5nd.com/2016/2016-11-15/74847/1.mp3");
            FileUtils.writeByteArrayToFile(new File("D:\test3_"+i+".mp3"), by1);
        }
        System.out.println(System.currentTimeMillis() - time1);
    }

结论：同样的文件，测试发现，在一定数量(文件数量大概15)的文件下载的时候，每个文件是3.6M的情况下，使用多线程大概时间是，而不实用多线程大概是，还是有一定时间上的提升

此处的测试还是会存在一定的误差，因为线程池的创建是需要时间，类的初始化也是需要一定的时间java并发写文件，假如在Web应用中，第一次创建出来了java并发写文件Java多线程下载文件，第二次不需要创建Java多线程下载文件，优势还是稍微能够体现出来！！