java http大文件斷點續傳上傳

1，項目調研

因為需要研究下斷點上傳的問題。找了很久終于找到一個比較好的項目。

在GoogleCode上面，代碼弄下來超級不方便，還是配置hosts才好，把代碼重新上傳到了github上面。

https://github.com/freewebsys/java-large-file-uploader-demo

效果：

上傳中，顯示進度，時間，百分比。

點擊【Pause】暫停，點擊【Resume】繼續。

2，代碼分析

原始項目：

https://code.google.com/p/java-large-file-uploader/

這個項目最后更新的時間是 2012 年，項目進行了封裝使用最簡單的方法實現了http的斷點上傳。

因為html5 里面有讀取文件分割文件的類庫，所以才可以支持斷點上傳，所以這個只能在html5 支持的瀏覽器上面展示。

同時，在js 和 java 同時使用 cr32 進行文件塊的校驗，保證數據上傳正確。

代碼在使用了最新的servlet 3.0 的api，使用了異步執行，監聽等方法。

上傳類UploadServlet

    @Component("javaLargeFileUploaderServlet")

    @WebServlet(name = "javaLargeFileUploaderServlet", urlPatterns = { "/javaLargeFileUploaderServlet" })

    public class UploadServlet extends HttpRequestHandlerServlet

            implements HttpRequestHandler {


    private static final Logger log = LoggerFactory.getLogger(UploadServlet.class);  

    @Autowired  
    UploadProcessor uploadProcessor;  

    @Autowired  
    FileUploaderHelper fileUploaderHelper;  

    @Autowired  
    ExceptionCodeMappingHelper exceptionCodeMappingHelper;  

    @Autowired  
    Authorizer authorizer;  

    @Autowired  
    StaticStateIdentifierManager staticStateIdentifierManager;  



    @Override  
    public void handleRequest(HttpServletRequest request, HttpServletResponse response)  
            throws IOException {  
        log.trace("Handling request");  

        Serializable jsonObject = null;  
        try {  
            // extract the action from the request  
            UploadServletAction actionByParameterName =  
                    UploadServletAction.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.action));  

            // check authorization  
            checkAuthorization(request, actionByParameterName);  

            // then process the asked action  
            jsonObject = processAction(actionByParameterName, request);  


            // if something has to be written to the response  
            if (jsonObject != null) {  
                fileUploaderHelper.writeToResponse(jsonObject, response);  
            }  

        }  
        // If exception, write it  
        catch (Exception e) {  
            exceptionCodeMappingHelper.processException(e, response);  
        }  

    }  


    private void checkAuthorization(HttpServletRequest request, UploadServletAction actionByParameterName)  
            throws MissingParameterException, AuthorizationException {  

        // check authorization  
        // if its not get progress (because we do not really care about authorization for get  
        // progress and it uses an array of file ids)  
        if (!actionByParameterName.equals(UploadServletAction.getProgress)) {  

            // extract uuid  
            final String fileIdFieldValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId, false);  

            // if this is init, the identifier is the one in parameter  
            UUID clientOrJobId;  
            String parameter = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false);  
            if (actionByParameterName.equals(UploadServletAction.getConfig) && parameter != null) {  
                clientOrJobId = UUID.fromString(parameter);  
            }  
            // if not, get it from manager  
            else {  
                clientOrJobId = staticStateIdentifierManager.getIdentifier();  
            }  


            // call authorizer  
            authorizer.getAuthorization(  
                    request,  
                    actionByParameterName,  
                    clientOrJobId,  
                    fileIdFieldValue != null ? getFileIdsFromString(fileIdFieldValue).toArray(new UUID[] {}) : null);  

        }  
    }  


    private Serializable processAction(UploadServletAction actionByParameterName, HttpServletRequest request)  
            throws Exception {  
        log.debug("Processing action " + actionByParameterName.name());  

        Serializable returnObject = null;  
        switch (actionByParameterName) {  
            case getConfig:  
                String parameterValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false);  
                returnObject =  
                        uploadProcessor.getConfig(  
                                parameterValue != null ? UUID.fromString(parameterValue) : null);  
                break;  
            case verifyCrcOfUncheckedPart:  
                returnObject = verifyCrcOfUncheckedPart(request);  
                break;  
            case prepareUpload:  
                returnObject = prepareUpload(request);  
                break;  
            case clearFile:  
                uploadProcessor.clearFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)));  
                break;  
            case clearAll:  
                uploadProcessor.clearAll();  
                break;  
            case pauseFile:  
                List<UUID> uuids = getFileIdsFromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId));  
                uploadProcessor.pauseFile(uuids);  
                break;  
            case resumeFile:  
                returnObject =  
                        uploadProcessor.resumeFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)));  
                break;  
            case setRate:  
                uploadProcessor.setUploadRate(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)),  
                        Long.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.rate)));  
                break;  
            case getProgress:  
                returnObject = getProgress(request);  
                break;  
        }  
        return returnObject;  
    }  


    List<UUID> getFileIdsFromString(String fileIds) {  
        String[] splittedFileIds = fileIds.split(",");  
        List<UUID> uuids = Lists.newArrayList();  
        for (int i = 0; i < splittedFileIds.length; i++) {  
            uuids.add(UUID.fromString(splittedFileIds[i]));  
        }   
        return uuids;  
    }  


    private Serializable getProgress(HttpServletRequest request)  
            throws MissingParameterException {  
        Serializable returnObject;  
        String[] ids =  
                new Gson()  
                        .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId), String[].class);  
        Collection<UUID> uuids = Collections2.transform(Arrays.asList(ids), new Function<String, UUID>() {  

            @Override  
            public UUID apply(String input) {  
                return UUID.fromString(input);  
            }  

        });  
        returnObject = Maps.newHashMap();  
        for (UUID fileId : uuids) {  
            try {  
                ProgressJson progress = uploadProcessor.getProgress(fileId);  
                ((HashMap<String, ProgressJson>) returnObject).put(fileId.toString(), progress);  
            }  
            catch (FileNotFoundException e) {  
                log.debug("No progress will be retrieved for " + fileId + " because " + e.getMessage());  
            }  
        }  
        return returnObject;  
    }  


    private Serializable prepareUpload(HttpServletRequest request)  
            throws MissingParameterException, IOException {  

        // extract file information  
        PrepareUploadJson[] fromJson =  
                new Gson()  
                        .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.newFiles), PrepareUploadJson[].class);  

        // prepare them  
        final HashMap<String, UUID> prepareUpload = uploadProcessor.prepareUpload(fromJson);  

        // return them  
        return Maps.newHashMap(Maps.transformValues(prepareUpload, new Function<UUID, String>() {  

            public String apply(UUID input) {  
                return input.toString();  
            };  
        }));  
    }  


    private Boolean verifyCrcOfUncheckedPart(HttpServletRequest request)  
            throws IOException, MissingParameterException, FileCorruptedException, FileStillProcessingException {  
        UUID fileId = UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId));  
        try {  
            uploadProcessor.verifyCrcOfUncheckedPart(fileId,  
                    fileUploaderHelper.getParameterValue(request, UploadServletParameter.crc));  
        }  
        catch (InvalidCrcException e) {  
            // no need to log this exception, a fallback behaviour is defined in the  
            // throwing method.  
            // but we need to return something!  
            return Boolean.FALSE;  
        }  
        return Boolean.TRUE;  
    }  
}  </pre><br />

異步上傳UploadServletAsync 
    @Component("javaLargeFileUploaderAsyncServlet")

    @WebServlet(name = "javaLargeFileUploaderAsyncServlet", urlPatterns = { "/javaLargeFileUploaderAsyncServlet" }, asyncSupported = true)

    public class UploadServletAsync extends HttpRequestHandlerServlet

            implements HttpRequestHandler {


    private static final Logger log = LoggerFactory.getLogger(UploadServletAsync.class);  

    @Autowired  
    ExceptionCodeMappingHelper exceptionCodeMappingHelper;  

    @Autowired  
    UploadServletAsyncProcessor uploadServletAsyncProcessor;  

    @Autowired  
    StaticStateIdentifierManager staticStateIdentifierManager;  

    @Autowired  
    StaticStateManager<StaticStatePersistedOnFileSystemEntity> staticStateManager;  

    @Autowired  
    FileUploaderHelper fileUploaderHelper;  

    @Autowired  
    Authorizer authorizer;  

    /** 
     * Maximum time that a streaming request can take.<br> 
     */  
    private long taskTimeOut = DateUtils.MILLIS_PER_HOUR;  


    @Override  
    public void handleRequest(final HttpServletRequest request, final HttpServletResponse response)  
            throws ServletException, IOException {  

        // process the request  
        try {  

            //check if uploads are allowed  
            if (!uploadServletAsyncProcessor.isEnabled()) {  
                throw new UploadIsCurrentlyDisabled();  
            }  

            // extract stuff from request  
            final FileUploadConfiguration process = fileUploaderHelper.extractFileUploadConfiguration(request);  

            log.debug("received upload request with config: "+process);  

            // verify authorization  
            final UUID clientId = staticStateIdentifierManager.getIdentifier();  
            authorizer.getAuthorization(request, UploadServletAction.upload, clientId, process.getFileId());  

            //check if that file is not paused  
            if (uploadServletAsyncProcessor.isFilePaused(process.getFileId())) {  
                log.debug("file "+process.getFileId()+" is paused, ignoring async request.");  
                return;  
            }  

            // get the model  
            StaticFileState fileState = staticStateManager.getEntityIfPresent().getFileStates().get(process.getFileId());  
            if (fileState == null) {  
                throw new FileNotFoundException("File with id " + process.getFileId() + " not found");  
            }  

            // process the request asynchronously  
            final AsyncContext asyncContext = request.startAsync();  
            asyncContext.setTimeout(taskTimeOut);  


            // add a listener to clear bucket and close inputstream when process is complete or  
            // with  
            // error  
            asyncContext.addListener(new UploadServletAsyncListenerAdapter(process.getFileId()) {  

                @Override  
                void clean() {  
                    log.debug("request " + request + " completed.");  
                    // we do not need to clear the inputstream here.  
                    // and tell processor to clean its shit!  
                    uploadServletAsyncProcessor.clean(clientId, process.getFileId());  
                }  
            });  

            // then process  
            uploadServletAsyncProcessor.process(fileState, process.getFileId(), process.getCrc(), process.getInputStream(),  
                    new WriteChunkCompletionListener() {  

                        @Override  
                        public void success() {  
                            asyncContext.complete();  
                        }  


                        @Override  
                        public void error(Exception exception) {  
                            // handles a stream ended unexpectedly , it just means the user has  
                            // stopped the  
                            // stream  
                            if (exception.getMessage() != null) {  
                                if (exception.getMessage().equals("Stream ended unexpectedly")) {  
                                    log.warn("User has stopped streaming for file " + process.getFileId());  
                                }  
                                else if (exception.getMessage().equals("User cancellation")) {  
                                    log.warn("User has cancelled streaming for file id " + process.getFileId());  
                                    // do nothing  
                                }  
                                else {  
                                    exceptionCodeMappingHelper.processException(exception, response);  
                                }  
                            }  
                            else {  
                                exceptionCodeMappingHelper.processException(exception, response);  
                            }  

                            asyncContext.complete();  
                        }  

                    });  
        }  
        catch (Exception e) {  
            exceptionCodeMappingHelper.processException(e, response);  
        }  

    }  

}  </pre><br />

3，請求流程圖： 



主要思路就是將文件切分，然后分塊上傳。