問題:javascript
公司如今的業務需求是要上傳一個大文件,上一次寫了一篇博客,作了一個簡單的文件上傳,支持單文件,大型文件上傳css
如今對以前的上傳進行優化,支持斷點續傳,秒傳功能html
上次博客:【http://www.cnblogs.com/hackxiyu/p/8194066.html】前端
分析:java
這篇文章參考了其它博主的文章,參考地址:【https://github.com/Fourwenwen/Breakpoint-http】jquery
環境須要:git
1.本地測試的話須要配置好Redis,用來保存文件的MD5校驗值,和秒傳功能的實現github
2.jquery,bootstrap,webUploader的相關js,css文件web
3.我用的是springBoot來實現的,頁面是首頁嵌套的,因此沒有html,body標籤,你們根據本身狀況來定redis
解決:
1.頁面html文件,業務js文件雜糅到一塊兒,你們能夠拆開清晰一些
<!--引入css文件--> <link rel="stylesheet" type="text/css" href="static/html/bigFileUpload/assets/bootstrap-3.3.7-dist/css/bootstrap.css"> <link rel="stylesheet" type="text/css" href="static/html/bigFileUpload/assets/webuploader.css"> <div id="uploader" class="wu-example"> <div id="thelist" class="uploader-list"></div> <div class="btns"> <div id="picker">選擇大文件</div> <button id="ctlBtn" class="btn btn-default">開始上傳</button> </div> </div> <!--引入JS,jquery的js已經引入--> <script type="text/javascript" src="static/html/bigFileUpload/assets/webuploader.js"></script> <script type="text/javascript" src="static/html/bigFileUpload/assets/bootstrap-3.3.7-dist/js/bootstrap.js"></script> <!--業務js文件--> <script> var $btn = $('#ctlBtn'); var $thelist = $('#thelist'); var chunkSize = 5 * 1024 * 1024; // HOOK 這個必需要再uploader實例化前面 WebUploader.Uploader.register({ 'before-send-file': 'beforeSendFile', 'before-send': 'beforeSend' }, { beforeSendFile: function (file) { console.log("beforeSendFile"); // Deferred對象在鉤子回掉函數中常常要用到,用來處理須要等待的異步操做。 var task = new $.Deferred(); // 根據文件內容來查詢MD5 uploader.md5File(file).progress(function (percentage) { // 及時顯示進度 console.log('計算md5進度:', percentage); getProgressBar(file, percentage, "MD5", "MD5"); }).then(function (val) { // 完成 console.log('md5 result:', val); file.md5 = val; // 模擬用戶id // file.uid = new Date().getTime() + "_" + Math.random() * 100; file.uid = WebUploader.Base.guid(); // 進行md5判斷 $.post("break/checkFileMd5", {uid: file.uid, md5: file.md5,"Authorization": localStorage.token}, function (data) { console.log(data.status); var status = data.status.value; task.resolve(); if (status == 101) { // 文件不存在,那就正常流程 } else if (status == 100) { // 忽略上傳過程,直接標識上傳成功; uploader.skipFile(file); file.pass = true; } else if (status == 102) { // 部分已經上傳到服務器了,可是差幾個模塊。 file.missChunks = data.data; } }); }); return $.when(task); }, beforeSend: function (block) { console.log("block") var task = new $.Deferred(); var file = block.file; var missChunks = file.missChunks; var blockChunk = block.chunk; console.log("當前分塊:" + blockChunk); console.log("missChunks:" + missChunks); if (missChunks !== null && missChunks !== undefined && missChunks !== '') { var flag = true; for (var i = 0; i < missChunks.length; i++) { if (blockChunk == missChunks[i]) { console.log(file.name + ":" + blockChunk + ":還沒上傳,如今上傳去吧。"); flag = false; break; } } if (flag) { task.reject(); } else { task.resolve(); } } else { task.resolve(); } return $.when(task); } }); // 實例化 var uploader = WebUploader.create({ pick: { id: '#picker', label: '點擊選擇文件' }, formData: { uid: 0, md5: '', chunkSize: chunkSize, "Authorization": localStorage.token }, //dnd: '#dndArea', //paste: '#uploader', swf: 'static/html/bigFileUpload/assets/Uploader.swf', chunked: true, chunkSize: chunkSize, // 字節 1M分塊 threads: 3, server: 'break/fileUpload', auto: false, // 禁掉全局的拖拽功能。這樣不會出現圖片拖進頁面的時候,把圖片打開。 disableGlobalDnd: true, fileNumLimit: 1024, fileSizeLimit: 1024 * 1024 * 1024, // 200 M fileSingleSizeLimit: 1024 * 1024 * 1024 // 50 M }); // 當有文件被添加進隊列的時候 uploader.on('fileQueued', function (file) { console.log("fileQueued"); $thelist.append('<div id="' + file.id + '" class="item">' + '<h4 class="info">' + file.name + '</h4>' + '<p class="state">等待上傳...</p>' + '</div>'); }); //當某個文件的分塊在發送前觸發,主要用來詢問是否要添加附帶參數,大文件在開起分片上傳的前提下此事件可能會觸發屢次。 uploader.onUploadBeforeSend = function (obj, data) { console.log("onUploadBeforeSend"); var file = obj.file; data.md5 = file.md5 || ''; data.uid = file.uid; }; // 上傳中 uploader.on('uploadProgress', function (file, percentage) { getProgressBar(file, percentage, "FILE", "上傳進度"); }); // 上傳返回結果 uploader.on('uploadSuccess', function (file) { var text = '已上傳'; if (file.pass) { text = "文件妙傳功能,文件已上傳。" } $('#' + file.id).find('p.state').text(text); }); uploader.on('uploadError', function (file) { $('#' + file.id).find('p.state').text('上傳出錯'); }); uploader.on('uploadComplete', function (file) { // 隱藏進度條 fadeOutProgress(file, 'MD5'); fadeOutProgress(file, 'FILE'); }); // 文件上傳 $btn.on('click', function () { console.log("上傳..."); uploader.upload(); console.log("上傳成功"); }); /** * 生成進度條封裝方法 * @param file 文件 * @param percentage 進度值 * @param id_Prefix id前綴 * @param titleName 標題名 */ function getProgressBar(file, percentage, id_Prefix, titleName) { var $li = $('#' + file.id), $percent = $li.find('#' + id_Prefix + '-progress-bar'); // 避免重複建立 if (!$percent.length) { $percent = $('<div id="' + id_Prefix + '-progress" class="progress progress-striped active">' + '<div id="' + id_Prefix + '-progress-bar" class="progress-bar" role="progressbar" style="width: 0%">' + '</div>' + '</div>' ).appendTo($li).find('#' + id_Prefix + '-progress-bar'); } var progressPercentage = percentage * 100 + '%'; $percent.css('width', progressPercentage); $percent.html(titleName + ':' + progressPercentage); } /** * 隱藏進度條 * @param file 文件對象 * @param id_Prefix id前綴 */ function fadeOutProgress(file, id_Prefix) { $('#' + file.id).find('#' + id_Prefix + '-progress').fadeOut(); } </script>
2.API接口
package org.triber.portal.breakPoint; import org.apache.commons.io.FileUtils; import org.apache.tomcat.util.http.fileupload.servlet.ServletFileUpload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import javax.servlet.http.HttpServletRequest; import java.io.File; import java.io.IOException; import java.util.LinkedList; import java.util.List; /** * 斷點續傳上傳大文件類 */ @Controller @RequestMapping(value = "/break") public class BreakPointController { private Logger logger = LoggerFactory.getLogger(BreakPointController.class); @Autowired private StringRedisTemplate stringRedisTemplate; @Autowired private StorageService storageService; /** * 秒傳判斷,斷點判斷 * * @return */ @RequestMapping(value = "checkFileMd5", method = RequestMethod.POST) @ResponseBody public Object checkFileMd5(String md5) throws IOException { Object processingObj = stringRedisTemplate.opsForHash().get(Constants.FILE_UPLOAD_STATUS, md5); if (processingObj == null) { return new ResultVo(ResultStatus.NO_HAVE); } String processingStr = processingObj.toString(); boolean processing = Boolean.parseBoolean(processingStr); String value = stringRedisTemplate.opsForValue().get(Constants.FILE_MD5_KEY + md5); if (processing) { return new ResultVo(ResultStatus.IS_HAVE, value); } else { File confFile = new File(value); byte[] completeList = FileUtils.readFileToByteArray(confFile); List<String> missChunkList = new LinkedList<>(); for (int i = 0; i < completeList.length; i++) { if (completeList[i] != Byte.MAX_VALUE) { missChunkList.add(i + ""); } } return new ResultVo<>(ResultStatus.ING_HAVE, missChunkList); } } /** * 上傳文件 * * @param param * @param request * @return * @throws Exception */ @RequestMapping(value = "/fileUpload", method = RequestMethod.POST) @ResponseBody public ResponseEntity fileUpload(MultipartFileParam param, HttpServletRequest request) { boolean isMultipart = ServletFileUpload.isMultipartContent(request); if (isMultipart) { logger.info("上傳文件start。"); try { // 方法1 //storageService.uploadFileRandomAccessFile(param); // 方法2 這個更快點 storageService.uploadFileByMappedByteBuffer(param); } catch (IOException e) { e.printStackTrace(); logger.error("文件上傳失敗。{}", param.toString()); } logger.info("上傳文件end。"); } return ResponseEntity.ok().body("上傳成功。"); }
}
3.業務service的實現
package org.triber.portal.breakPoint; import java.io.IOException; /** * 存儲操做的service * Created by 超文 on 2017/5/2. */ public interface StorageService { /** * 刪除所有數據 */ void deleteAll(); /** * 初始化方法 */ void init(); /** * 上傳文件方法1 * * @param param * @throws IOException */ void uploadFileRandomAccessFile(MultipartFileParam param) throws IOException; /** * 上傳文件方法2 * 處理文件分塊,基於MappedByteBuffer來實現文件的保存 * * @param param * @throws IOException */ void uploadFileByMappedByteBuffer(MultipartFileParam param) throws IOException; }
實現
package org.triber.portal.breakPoint; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; import org.springframework.util.FileSystemUtils; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; /** * Created by 超文 on 2017/5/2. */ @Service public class StorageServiceImpl implements StorageService { private final Logger logger = LoggerFactory.getLogger(StorageServiceImpl.class); // 保存文件的根目錄 private Path rootPaht; @Autowired private StringRedisTemplate stringRedisTemplate; //這個必須與前端設定的值一致 @Value("${breakpoint.upload.chunkSize}") private long CHUNK_SIZE; @Value("${breakpoint.upload.dir}") private String finalDirPath; @Autowired public StorageServiceImpl(@Value("${breakpoint.upload.dir}") String location) { this.rootPaht = Paths.get(location); } @Override public void deleteAll() { logger.info("開發初始化清理數據,start"); FileSystemUtils.deleteRecursively(rootPaht.toFile()); stringRedisTemplate.delete(Constants.FILE_UPLOAD_STATUS); stringRedisTemplate.delete(Constants.FILE_MD5_KEY); logger.info("開發初始化清理數據,end"); } @Override public void init() { try { Files.createDirectory(rootPaht); } catch (FileAlreadyExistsException e) { logger.error("文件夾已經存在了,不用再建立。"); } catch (IOException e) { logger.error("初始化root文件夾失敗。", e); } } @Override public void uploadFileRandomAccessFile(MultipartFileParam param) throws IOException { String fileName = param.getName(); String tempDirPath = finalDirPath + param.getMd5(); String tempFileName = fileName + "_tmp"; File tmpDir = new File(tempDirPath); File tmpFile = new File(tempDirPath, tempFileName); if (!tmpDir.exists()) { tmpDir.mkdirs(); } RandomAccessFile accessTmpFile = new RandomAccessFile(tmpFile, "rw"); long offset = CHUNK_SIZE * param.getChunk(); //定位到該分片的偏移量 accessTmpFile.seek(offset); //寫入該分片數據 accessTmpFile.write(param.getFile().getBytes()); // 釋放 accessTmpFile.close(); boolean isOk = checkAndSetUploadProgress(param, tempDirPath); if (isOk) { boolean flag = renameFile(tmpFile, fileName); System.out.println("upload complete !!" + flag + " name=" + fileName); } } @Override public void uploadFileByMappedByteBuffer(MultipartFileParam param) throws IOException { String fileName = param.getName(); String uploadDirPath = finalDirPath + param.getMd5(); String tempFileName = fileName + "_tmp"; File tmpDir = new File(uploadDirPath); File tmpFile = new File(uploadDirPath, tempFileName); if (!tmpDir.exists()) { tmpDir.mkdirs(); } RandomAccessFile tempRaf = new RandomAccessFile(tmpFile, "rw"); FileChannel fileChannel = tempRaf.getChannel(); //寫入該分片數據 long offset = CHUNK_SIZE * param.getChunk(); byte[] fileData = param.getFile().getBytes(); MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, offset, fileData.length); mappedByteBuffer.put(fileData); // 釋放 FileMD5Util.freedMappedByteBuffer(mappedByteBuffer); fileChannel.close(); boolean isOk = checkAndSetUploadProgress(param, uploadDirPath); if (isOk) { boolean flag = renameFile(tmpFile, fileName); System.out.println("upload complete !!" + flag + " name=" + fileName); } } /** * 檢查並修改文件上傳進度 * * @param param * @param uploadDirPath * @return * @throws IOException */ private boolean checkAndSetUploadProgress(MultipartFileParam param, String uploadDirPath) throws IOException { String fileName = param.getName(); File confFile = new File(uploadDirPath, fileName + ".conf"); RandomAccessFile accessConfFile = new RandomAccessFile(confFile, "rw"); //把該分段標記爲 true 表示完成 System.out.println("set part " + param.getChunk() + " complete"); accessConfFile.setLength(param.getChunks()); accessConfFile.seek(param.getChunk()); accessConfFile.write(Byte.MAX_VALUE); //completeList 檢查是否所有完成,若是數組裏是否所有都是(所有分片都成功上傳) byte[] completeList = FileUtils.readFileToByteArray(confFile); byte isComplete = Byte.MAX_VALUE; for (int i = 0; i < completeList.length && isComplete == Byte.MAX_VALUE; i++) { //與運算, 若是有部分沒有完成則 isComplete 不是 Byte.MAX_VALUE isComplete = (byte) (isComplete & completeList[i]); System.out.println("check part " + i + " complete?:" + completeList[i]); } accessConfFile.close(); if (isComplete == Byte.MAX_VALUE) { stringRedisTemplate.opsForHash().put(Constants.FILE_UPLOAD_STATUS, param.getMd5(), "true"); stringRedisTemplate.opsForValue().set(Constants.FILE_MD5_KEY + param.getMd5(), uploadDirPath + "/" + fileName); return true; } else { if (!stringRedisTemplate.opsForHash().hasKey(Constants.FILE_UPLOAD_STATUS, param.getMd5())) { stringRedisTemplate.opsForHash().put(Constants.FILE_UPLOAD_STATUS, param.getMd5(), "false"); } if (stringRedisTemplate.hasKey(Constants.FILE_MD5_KEY + param.getMd5())) { stringRedisTemplate.opsForValue().set(Constants.FILE_MD5_KEY + param.getMd5(), uploadDirPath + "/" + fileName + ".conf"); } return false; } } /** * 文件重命名 * * @param toBeRenamed 將要修更名字的文件 * @param toFileNewName 新的名字 * @return */ public boolean renameFile(File toBeRenamed, String toFileNewName) { //檢查要重命名的文件是否存在,是不是文件 if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) { logger.info("File does not exist: " + toBeRenamed.getName()); return false; } String p = toBeRenamed.getParent(); File newFile = new File(p + File.separatorChar + toFileNewName); //修改文件名 return toBeRenamed.renameTo(newFile); } }
4.依賴的MD5工具類
package org.triber.portal.breakPoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.lang.reflect.Method; import java.math.BigInteger; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.security.AccessController; import java.security.MessageDigest; import java.security.PrivilegedAction; /** * 文件md5值 * Created by 超文 on 2016/10/10. * version 1.0 */ public class FileMD5Util { private final static Logger logger = LoggerFactory.getLogger(FileMD5Util.class); public static String getFileMD5(File file) throws FileNotFoundException { String value = null; FileInputStream in = new FileInputStream(file); MappedByteBuffer byteBuffer = null; try { byteBuffer = in.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()); MessageDigest md5 = MessageDigest.getInstance("MD5"); md5.update(byteBuffer); BigInteger bi = new BigInteger(1, md5.digest()); value = bi.toString(16); if (value.length() < 32) { value = "0" + value; } } catch (Exception e) { e.printStackTrace(); } finally { if (null != in) { try { in.getChannel().close(); in.close(); } catch (IOException e) { logger.error("get file md5 error!!!", e); } } if (null != byteBuffer) { freedMappedByteBuffer(byteBuffer); } } return value; } /** * 在MappedByteBuffer釋放後再對它進行讀操做的話就會引起jvm crash,在併發狀況下很容易發生 * 正在釋放時另外一個線程正開始讀取,因而crash就發生了。因此爲了系統穩定性釋放前通常須要檢 查是否還有線程在讀或寫 * * @param mappedByteBuffer */ public static void freedMappedByteBuffer(final MappedByteBuffer mappedByteBuffer) { try { if (mappedByteBuffer == null) { return; } mappedByteBuffer.force(); AccessController.doPrivileged(new PrivilegedAction<Object>() { @Override public Object run() { try { Method getCleanerMethod = mappedByteBuffer.getClass().getMethod("cleaner", new Class[0]); getCleanerMethod.setAccessible(true); sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(mappedByteBuffer, new Object[0]); cleaner.clean(); } catch (Exception e) { logger.error("clean MappedByteBuffer error!!!", e); } logger.info("clean MappedByteBuffer completed!!!"); return null; } }); } catch (Exception e) { e.printStackTrace(); } } }
5.分片實體
package org.triber.portal.breakPoint; import org.springframework.web.multipart.MultipartFile; /** * Created by wenwen on 2017/4/16. * version 1.0 */ public class MultipartFileParam { // 用戶id private String uid; //任務ID private String id; //總分片數量 private int chunks; //當前爲第幾塊分片 private int chunk; //當前分片大小 private long size = 0L; //文件名 private String name; //分片對象 private MultipartFile file; // MD5 private String md5; public String getUid() { return uid; } public void setUid(String uid) { this.uid = uid; } public String getId() { return id; } public void setId(String id) { this.id = id; } public int getChunks() { return chunks; } public void setChunks(int chunks) { this.chunks = chunks; } public int getChunk() { return chunk; } public void setChunk(int chunk) { this.chunk = chunk; } public long getSize() { return size; } public void setSize(long size) { this.size = size; } public String getName() { return name; } public void setName(String name) { this.name = name; } public MultipartFile getFile() { return file; } public void setFile(MultipartFile file) { this.file = file; } public String getMd5() { return md5; } public void setMd5(String md5) { this.md5 = md5; } @Override public String toString() { return "MultipartFileParam{" + "uid='" + uid + '\'' + ", id='" + id + '\'' + ", chunks=" + chunks + ", chunk=" + chunk + ", size=" + size + ", name='" + name + '\'' + ", file=" + file + ", md5='" + md5 + '\'' + '}'; } }
6.響應常量類
package org.triber.portal.breakPoint; import com.fasterxml.jackson.annotation.JsonFormat; /** * 結果類型枚舉 * Created by 超文 on 2017/5/2. * version 1.0 */ @JsonFormat(shape = JsonFormat.Shape.OBJECT) public enum ResultStatus { /** * 1 開頭爲判斷文件在系統的狀態 */ IS_HAVE(100, "文件已存在!"), NO_HAVE(101, "該文件沒有上傳過。"), ING_HAVE(102, "該文件上傳了一部分。"); private final int value; private final String reasonPhrase; ResultStatus(int value, String reasonPhrase) { this.value = value; this.reasonPhrase = reasonPhrase; } public int getValue() { return value; } public String getReasonPhrase() { return reasonPhrase; } }
7.響應實體
package org.triber.portal.breakPoint; /** * 統一返回結果pojo * Created by wenwen on 2017/4/23. * version 1.0 */ public class ResultVo<T> { private ResultStatus status; private String msg; private T data; public ResultVo(ResultStatus status) { this(status, status.getReasonPhrase(), null); } public ResultVo(ResultStatus status, T data) { this(status, status.getReasonPhrase(), data); } public ResultVo(ResultStatus status, String msg, T data) { this.status = status; this.msg = msg; this.data = data; } public ResultStatus getStatus() { return status; } public void setStatus(ResultStatus status) { this.status = status; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } public T getData() { return data; } public void setData(T data) { this.data = data; } @Override public String toString() { return "ResultVo{" + "status=" + status + ", msg='" + msg + '\'' + ", data=" + data + '}'; } }
8.常量類
package org.triber.portal.breakPoint; import java.util.HashMap; import java.util.Map; /** * 常量表 * Created by 超文 on 2017/05/02. * version 1.0 */ public interface Constants { /** * 異常信息統一頭信息<br> * 很是遺憾的通知您,程序發生了異常 */ public static final String Exception_Head = "boom。炸了。"; /** * 緩存鍵值 */ public static final Map<Class<?>, String> cacheKeyMap = new HashMap<>(); /** * 保存文件所在路徑的key,eg.FILE_MD5:1243jkalsjflkwaejklgjawe */ public static final String FILE_MD5_KEY = "FILE_MD5:"; /** * 保存上傳文件的狀態 */ public static final String FILE_UPLOAD_STATUS = "FILE_UPLOAD_STATUS"; }
9.本機Redis配置
#開發環境
breakpoint:
upload:
dir: E:/data0/uploads/
#1024*1024=1 048 576,5M=5 242 880
chunkSize: 5 242 880
spring:
redis:
host: 127.0.0.1
port: 6379
# password: test //密碼我本機沒有因此不配
pool:
max-active: 30
max-idle: 10
max-wait: 10000
timeout: 0
http:
multipart:
max-file-size: 10MB //能夠自定義這些值 max-request-size: 100MB
總結:
其實重要的也就是,頁面js文件,和後臺接口服務,MD5工具類