硬盤緩存方案DiskLruCache源碼解析
來自: http://blog.csdn.net/cauchyweierstrass/article/details/50687778
前面研究了LruCache,它作為現在用的最多的內存緩存方案已經在很多開源緩存框架中使用,同樣的還有硬盤緩存方案也就是DiskLruCache,通常的做法就是使用內存和硬盤二級緩存。
使用方法
1.存儲:
DiskLruCache diskLruCache= open(File directory, int appVersion, int valueCount, long maxSize); DiskLruCache.Editor editor = diskLruCache.edit(key); OuputStream ouputStream = editor.newOutputStream(0);
然后往該ouputStream中寫入收即可。如果是寫入字符串可以使用,其實就是封裝了往輸出流中寫的代碼。
editor.set(int index, String value);
2.訪問
DiskLruCache.Snapshot snapShot = mDiskLruCache.get(key); if (snapShot != null) { InputStream is = snapShot.getInputStream(0); }
如果是文本,可以直接獲取
snapShot.getString(int index)
3.close() 和open方法對應,用于關閉DiskLruCache里面的方法,journalWriter被置null,就不能再更新日志文件了。
4.delete() 刪除緩存目錄下的所有文件,用于清空緩存。
二級緩存的框架
綜合前面的內存緩存LruCache和硬盤緩存DiskLruCache,內存和硬盤二級緩存的大概框架可以如下代碼
Bitmap bitmap = getBitmap(generateKey(url)) if (bitmap == null) { downLoadFromNetOnAsyncTask(url); // set the bitmap from menory }Bitmap getBitmap(String key){ Bitmap bitmap = null; if ((bitmap = lruCache.get(key)) == null) { DiskLruCache.Snapshot snapShot = diskLruCache.get(key);
if (snapShot != null) {
InputStream is = snapShot.getInputStream(0);
bitmap = BitmapFactory.decodeStream(is); lruCache.put(bitmap); } } return bitmap; } void downLoadFromNetOnAsyncTask(String url) { // download a picture via Thread DiskLruCache.Editor editor = diskLruCache.edit(generateKey(url)); OuputStream ouputStream = editor.newOutputStream(0); storeToMemoryAndDisk(); }</pre>日志文件格式
This cache uses a journal file named "journal". A typical journal file
looks like this:
libcore.io.DiskLruCache
1
100
2
CLEAN 3400330d1dfc7f3f7f4b8d4d803dfcf6 832 21054
DIRTY 335c4c6028171cfddfbaae1a9c313c52
CLEAN 335c4c6028171cfddfbaae1a9c313c52 3934 2342
REMOVE 335c4c6028171cfddfbaae1a9c313c52
DIRTY 1ab96a171faeeee38496d8b330771a7a
CLEAN 1ab96a171faeeee38496d8b330771a7a 1600 234
READ 335c4c6028171cfddfbaae1a9c313c52
READ 3400330d1dfc7f3f7f4b8d4d803dfcf6
The first five lines of the journal form its header. They are the constant string "libcore.io.DiskLruCache", the disk cache's version,the application's version, the value count, and a blank line.
Each of the subsequent lines in the file is a record of the state of a cache entry. Each line contains space-separated values: a state, a key,and optional state-specific values.
- DIRTY lines track that an entry is actively being created or updated.Every successful DIRTY action should be followed by a CLEAN or REMOVE action. DIRTY lines without a matching CLEAN or REMOVE indicate thattemporary files may need to be deleted.
- CLEAN lines track a cache entry that has been successfully published and may be read. A publish line is followed by the lengths of each of its values.
- READ lines track accesses for LRU.
- REMOVE lines track entries that have been deleted.
源碼分析
public final class DiskLruCache implements Closeable { static final String JOURNAL_FILE = "journal"; static final String JOURNAL_FILE_TMP = "journal.tmp"; static final String MAGIC = "libcore.io.DiskLruCache"; static final String VERSION_1 = "1"; static final long ANY_SEQUENCE_NUMBER = -1; private static final String CLEAN = "CLEAN"; private static final String DIRTY = "DIRTY"; private static final String REMOVE = "REMOVE"; private static final String READ = "READ";private final File directory; private final File journalFile; private final File journalFileTmp; private final int appVersion; private final long maxSize; // 一個鍵對應幾個文件 private final int valueCount; private long size = 0; // 全局操作日志文件 private Writer journalWriter; private final LinkedHashMap<String, Entry> lruEntries = new LinkedHashMap<String, Entry>(0, 0.75f, true); private int redundantOpCount; // 用來標識被成功提交的序號 private long nextSequenceNumber = 0; // 用一個線程來處理在冗余操作大于2000后重構日志 private final ExecutorService executorService = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); private final Callable<Void> cleanupCallable = new Callable<Void>() { @Override public Void call() throws Exception { synchronized (DiskLruCache.this) { if (journalWriter == null) { return null; // closed } trimToSize(); if (journalRebuildRequired()) { rebuildJournal(); redundantOpCount = 0; } } return null; } }; private DiskLruCache(File directory, int appVersion, int valueCount, long maxSize) { this.directory = directory; this.appVersion = appVersion; this.journalFile = new File(directory, JOURNAL_FILE); this.journalFileTmp = new File(directory, JOURNAL_FILE_TMP); this.valueCount = valueCount; this.maxSize = maxSize; } / public static DiskLruCache open(File directory, int appVersion, int valueCount, long maxSize) throws IOException { if (maxSize <= 0) { throw new IllegalArgumentException("maxSize <= 0"); } if (valueCount <= 0) { throw new IllegalArgumentException("valueCount <= 0"); } // prefer to pick up where we left off DiskLruCache cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); // 日志文件存在即解析 if (cache.journalFile.exists()) { try { cache.readJournal(); cache.processJournal(); cache.journalWriter = new BufferedWriter(new FileWriter(cache.journalFile, true)); return cache; } catch (IOException journalIsCorrupt) { Libcore.logW("DiskLruCache " + directory + " is corrupt: " + journalIsCorrupt.getMessage() + ", removing"); cache.delete(); } } // 不存在則創建新的日志文件,然后重建日志文件。 directory.mkdirs(); cache = new DiskLruCache(directory, appVersion, valueCount, maxSize); cache.rebuildJournal(); return cache; } // 讀日志文件 private void readJournal() throws IOException { InputStream in = new BufferedInputStream(new FileInputStream(journalFile)); try { String magic = Streams.readAsciiLine(in); String version = Streams.readAsciiLine(in); String appVersionString = Streams.readAsciiLine(in); String valueCountString = Streams.readAsciiLine(in); String blank = Streams.readAsciiLine(in); if (!MAGIC.equals(magic) || !VERSION_1.equals(version) || !Integer.toString(appVersion).equals(appVersionString) || !Integer.toString(valueCount).equals(valueCountString) || !"".equals(blank)) { throw new IOException("unexpected journal header: [" + magic + ", " + version + ", " + valueCountString + ", " + blank + "]"); } while (true) { try { // 循環讀取每一行 readJournalLine(Streams.readAsciiLine(in)); } catch (EOFException endOfJournal) { break; } } } finally { IoUtils.closeQuietly(in); } } // 讀每一行,根據每行的字符串構建Entry private void readJournalLine(String line) throws IOException { String[] parts = line.split(" "); if (parts.length < 2) { throw new IOException("unexpected journal line: " + line); } String key = parts[1]; if (parts[0].equals(REMOVE) && parts.length == 2) { lruEntries.remove(key); return; } // 如果存在對key的操作會使其移動到鏈表尾 Entry entry = lruEntries.get(key); if (entry == null) { // 如果不存在則添加 entry = new Entry(key); lruEntries.put(key, entry); } if (parts[0].equals(CLEAN) && parts.length == 2 + valueCount) { entry.readable = true; entry.currentEditor = null; entry.setLengths(Arrays.copyOfRange(parts, 2, parts.length)); } else if (parts[0].equals(DIRTY) && parts.length == 2) { entry.currentEditor = new Editor(entry); } else if (parts[0].equals(READ) && parts.length == 2) { // this work was already done by calling lruEntries.get() // 如果為READ則什么都不需要做。上面這句翻譯一下就是說這里要做的工作已經在調用lruEntries.get()時做過了 // 遇到READ其實就是再次訪問該key,因此上面調用get的時候已經將其移動到最近使用的位置了 } else { throw new IOException("unexpected journal line: " + line); } } // 處理日志,將添加到Map中的所有鍵所占的磁盤空間加起來。賦值給size private void processJournal() throws IOException { // 如果有日志文件的備份文件存在則刪除它 deleteIfExists(journalFileTmp); for (Iterator<Entry> i = lruEntries.values().iterator(); i.hasNext();) { Entry entry = i.next(); if (entry.currentEditor == null) { for (int t = 0; t < valueCount; t++) { size += entry.lengths[t]; } } else { // 當前條目正在被編輯,刪除正在編輯的文件并將currentEditor賦值為null entry.currentEditor = null; for (int t = 0; t < valueCount; t++) { deleteIfExists(entry.getCleanFile(t)); deleteIfExists(entry.getDirtyFile(t)); } i.remove(); } } } // 創建新的日志文件,忽略掉冗余的信息,也就是對hashmap里面的元素進行遍歷重新寫日志文件 private synchronized void rebuildJournal() throws IOException { if (journalWriter != null) { journalWriter.close(); } // 創建臨時日志文件 Writer writer = new BufferedWriter(new FileWriter(journalFileTmp)); writer.write(MAGIC); writer.write("\n"); writer.write(VERSION_1); writer.write("\n"); writer.write(Integer.toString(appVersion)); writer.write("\n"); writer.write(Integer.toString(valueCount)); writer.write("\n"); writer.write("\n"); // 遍歷Map寫入日志文件 for (Entry entry : lruEntries.values()) { if (entry.currentEditor != null) { writer.write(DIRTY + ' ' + entry.key + '\n'); } else { writer.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); } } writer.close(); // 重命名 journalFileTmp.renameTo(journalFile); journalWriter = new BufferedWriter(new FileWriter(journalFile, true)); } private static void deleteIfExists(File file) throws IOException { Libcore.deleteIfExists(file); } // 獲取某個key對應的快照,通過該快照可以恢復到該緩存的文件 public synchronized Snapshot get(String key) throws IOException { checkNotClosed(); validateKey(key); // 對它的訪問會讓他移動到Map的最近使用過的位置 Entry entry = lruEntries.get(key); if (entry == null) { return null; } if (!entry.readable) { return null; } /* * Open all streams eagerly to guarantee that we see a single published * snapshot. If we opened streams lazily then the streams could come * from different edits. */ InputStream[] ins = new InputStream[valueCount]; try { for (int i = 0; i < valueCount; i++) { // 通過干凈的文件創建的流,于是如果獲取某個鍵對應的文件還未被編輯完成,文件不存在, // 那么返回null,因此可以在從硬盤獲取時根據返回值是否為null判斷硬盤是否有該緩存 ins[i] = new FileInputStream(entry.getCleanFile(i)); } } catch (FileNotFoundException e) { // a file must have been deleted manually! return null; } redundantOpCount++; journalWriter.append(READ + ' ' + key + '\n'); if (journalRebuildRequired()) { executorService.submit(cleanupCallable); } return new Snapshot(key, entry.sequenceNumber, ins); } /** * Returns an editor for the entry named {@code key}, or null if another * edit is in progress. */ public Editor edit(String key) throws IOException { return edit(key, ANY_SEQUENCE_NUMBER); } // 當DiskLruCache類調用edit方法傳入的都是ANY_SEQUENCE_NUMBER,Snapshot調用edit的時候入當前Entry的序列號 // 目的是當Snapshot調用edit的時候如果該條目的序列號已經改變了(在持有這個Snapshot后又成功commit了)就會返回null // 這也就是為什么Snapshot命名為快照的含義。 private synchronized Editor edit(String key, long expectedSequenceNumber) throws IOException { checkNotClosed(); validateKey(key); Entry entry = lruEntries.get(key); if (expectedSequenceNumber != ANY_SEQUENCE_NUMBER && (entry == null || entry.sequenceNumber != expectedSequenceNumber)) { return null; // snapshot is stale } if (entry == null) { entry = new Entry(key); // 只要edit,就put進去,這樣在commit的時候如果出錯就lrucache.remove,并且添加REMOVE記錄。 // 成功不用再put。除了put進HashMap中的記錄,其余的全部屬于冗余的。其他版本用總行數-lrucache.size lruEntries.put(key, entry); } else if (entry.currentEditor != null) { return null; // another edit is in progress } Editor editor = new Editor(entry); entry.currentEditor = editor; // 寫入一條臟數記錄 journalWriter.write(DIRTY + ' ' + key + '\n'); journalWriter.flush(); return editor; } public File getDirectory() { return directory; } public long maxSize() { return maxSize; } public synchronized long size() { return size; } private synchronized void completeEdit(Editor editor, boolean success) throws IOException { Entry entry = editor.entry; if (entry.currentEditor != editor) { throw new IllegalStateException(); } // readabe為false也即該條目還未被成功寫入成為CLEAN,也就是首次創建緩存文件 // 也就是CLEAN必須和DIRTY配對,如果臟的文件不存在就出現異常edit didn't create file if (success && !entry.readable) { for (int i = 0; i < valueCount; i++) { if (!entry.getDirtyFile(i).exists()) { editor.abort(); // 一定要在同一個線程里面創建newOutputStream和提交commit,不然可能會出問題 throw new IllegalStateException("edit didn't create file " + i); } } } // 如果sucess也就是寫I/O未出現異常那么將臟的文件重命名成干凈的文件然后更新entry的中的文件大小字段,并且更新已經占用的磁盤空間size。 // 如果false也就是寫I/O出錯,就將臟的文件刪除 for (int i = 0; i < valueCount; i++) { File dirty = entry.getDirtyFile(i); if (success) { if (dirty.exists()) { File clean = entry.getCleanFile(i); dirty.renameTo(clean); long oldLength = entry.lengths[i]; long newLength = clean.length(); entry.lengths[i] = newLength; size = size - oldLength + newLength; } } else { // 將臟的文件刪除 deleteIfExists(dirty); } } redundantOpCount++; // 提交完成后不論成功與否,將該條目的currentEditor置為null,以便其他地方可以在同一個key上進行再次edit entry.currentEditor = null; // readable為true指這個文件已經被成功創建過了。 // 有兩地方對readable賦值為true。一個是在解析日志文件時遇到clean時,此時即在再次訪問該條目,要寫入到日志文件 // 另一處就是這里,如果寫入成功了success,那么將置為readble置為true,此時還要寫入到日志文件 // 這里條件指要么之前該條目已經被成功的寫入過,或者這次成功寫入或者都為true if (entry.readable | success) { entry.readable = true; journalWriter.write(CLEAN + ' ' + entry.key + entry.getLengths() + '\n'); if (success) { // 每次提交成功就重新賦值該Entry的序列號加一。 entry.sequenceNumber = nextSequenceNumber++; } } else { // 首次寫入并且寫入失敗 lruEntries.remove(entry.key); journalWriter.write(REMOVE + ' ' + entry.key + '\n'); } // 判斷是否達到重構日志的條件,執行之。 if (size > maxSize || journalRebuildRequired()) { // 在一個新的線程里面重構日志 executorService.submit(cleanupCallable); } } // 判斷是否達到重構日志的要求:冗余操作大于有效數據的數目并且大于2000 private boolean journalRebuildRequired() { final int redundantOpCompactThreshold = 2000; return redundantOpCount >= redundantOpCompactThreshold && redundantOpCount >= lruEntries.size(); } /** * Drops the entry for {@code key} if it exists and can be removed. Entries * actively being edited cannot be removed. * * @return true if an entry was removed. */ public synchronized boolean remove(String key) throws IOException { checkNotClosed(); validateKey(key); Entry entry = lruEntries.get(key); // 正處于編輯狀態還未被提交則不能被remove if (entry == null || entry.currentEditor != null) { return false; } // 循環刪除對應的文件 for (int i = 0; i < valueCount; i++) { File file = entry.getCleanFile(i); if (!file.delete()) { throw new IOException("failed to delete " + file); } size -= entry.lengths[i]; entry.lengths[i] = 0; } redundantOpCount++; // 添加REMOVE記錄 journalWriter.append(REMOVE + ' ' + key + '\n'); // 從LinkedHashMap中移除 lruEntries.remove(key); // 判斷重構條件 if (journalRebuildRequired()) { executorService.submit(cleanupCallable); } return true; } public boolean isClosed() { return journalWriter == null; } private void checkNotClosed() { if (journalWriter == null) { throw new IllegalStateException("cache is closed"); } } public synchronized void flush() throws IOException { checkNotClosed(); trimToSize(); journalWriter.flush(); } // 關閉DiskLruCache public synchronized void close() throws IOException { if (journalWriter == null) { return; // already closed } // 終止所有的正處于編輯狀態的條目 for (Entry entry : new ArrayList<Entry>(lruEntries.values())) { if (entry.currentEditor != null) { entry.currentEditor.abort(); } } // 刪除最近最少使用的條目 trimToSize(); journalWriter.close(); journalWriter = null; } // 刪除最近最少使用的條目,包括從Map刪除和從磁盤刪除文件 private void trimToSize() throws IOException { while (size > maxSize) { Map.Entry<String, Entry> toEvict = lruEntries.entrySet().iterator().next(); remove(toEvict.getKey()); } } // 刪除所有的緩存文件,清除緩存用 public void delete() throws IOException { close(); IoUtils.deleteContents(directory); } private void validateKey(String key) { if (key.contains(" ") || key.contains("\n") || key.contains("\r")) { throw new IllegalArgumentException( "keys must not contain spaces or newlines: \"" + key + "\""); } } private static String inputStreamToString(InputStream in) throws IOException { return Streams.readFully(new InputStreamReader(in, Charsets.UTF_8)); } // DiskLruCache的get方法返回的對象的封裝 // 其實和Entry幾乎一樣的。為什么要從新封裝一個Snapshot類呢? // 1.這個類里面封裝了對應的緩存的文件的流,通過該流可以讀取到里面的數據 // 2.這里面封裝的sequenceNumber用于判斷該快照是否已經過期,獲取到該快照后,如果該key又經過了commit, // 它的序列號已經增加,于是該快照就過期了,調用edit方法返回null public final class Snapshot implements Closeable { // 和Entry中的key一樣 private final String key; // 顧名思義是序列號的意思,這個值是在調用get方法獲得快照的時候從Entry里面讀取的。 private final long sequenceNumber; // 對應的輸入流的數組,通過它可以讀取到文件里面的數據 private final InputStream[] ins; private Snapshot(String key, long sequenceNumber, InputStream[] ins) { this.key = key; this.sequenceNumber = sequenceNumber; this.ins = ins; } // 通過快照過去對該條目的editor,可以有下面情況 // 1.如果是已經成功編輯完的則獲取一個新的editor // 2.該快照在創建后已經被改變了,返回null // 3.另一個editor正在編輯還未commit,返回null public Editor edit() throws IOException { return DiskLruCache.this.edit(key, sequenceNumber); } /** * Returns the unbuffered stream with the value for {@code index}. */ // 獲取xxx.index文件對應的流 public InputStream getInputStream(int index) { return ins[index]; } // 如果某個index的流里面是文本,則通過getString獲得文本 public String getString(int index) throws IOException { return inputStreamToString(getInputStream(index)); } @Override public void close() { for (InputStream in : ins) { IoUtils.closeQuietly(in); } } } // 通過DiskLruCache的edit方法獲取該Editor對象,用來完成對Entry的編輯 public final class Editor { // 每個editor編輯一個entry條目 private final Entry entry; // 標識在I/O過程中是否有錯誤發生 private boolean hasErrors; private Editor(Entry entry) { this.entry = entry; } // 獲取某個xxx.index文件的流,即用來讀取改文件 public InputStream newInputStream(int index) throws IOException { synchronized (DiskLruCache.this) { if (entry.currentEditor != this) { throw new IllegalStateException(); } if (!entry.readable) { return null; } return new FileInputStream(entry.getCleanFile(index)); } } // 以String的方式獲取最上一次提交的值,也就是獲取該條目中的xxx.index文件的內容。 // 一般用于改文件保存字符串時使用 public String getString(int index) throws IOException { InputStream in = newInputStream(index); return in != null ? inputStreamToString(in) : null; } // 返回一個OutputStream,也就是被封裝成FaultHidingOutputStream,不會再拋出I/O異常 // The returned output stream does not throw IOExceptions. public OutputStream newOutputStream(int index) throws IOException { synchronized (DiskLruCache.this) { if (entry.currentEditor != this) { throw new IllegalStateException(); } // 注意這里用的是getDirtyFile return new FaultHidingOutputStream(new FileOutputStream(entry.getDirtyFile(index))); } } // 用于往xx.index.tmp文件里面寫文本 // 一般用于改文件保存字符串時使用 // 其實只是在上面newOutputStream的基礎上增加了往文件里面寫數據的代碼,畢竟寫幾個字符串簡單嘛 // 如果是從網絡獲取較大的圖片或者文件,還是自己拿到這個OutputStream然后網里面寫比較好 public void set(int index, String value) throws IOException { Writer writer = null; try { writer = new OutputStreamWriter(newOutputStream(index), Charsets.UTF_8); writer.write(value); } finally { IoUtils.closeQuietly(writer); } } /** * Commits this edit so it is visible to readers. This releases the * edit lock so another edit may be started on the same key. */ // 每次edit后寫完數據需要調用commit函數 // 調用commit的原因: // 1.根據流的寫過程是否出錯調用completeEdit函數執行不同的邏輯 // 2.在completeEdit函數里最重要的一點是entry.currentEditor = null;另一個edit才能在同一個key上編輯寫入,這一點參考323行,每次調用edit方法如果在該key對應的條目的currentEditor部位null說明有一個edit正在編輯它,就要返回null,即現在不允許在對他進行編輯 public void commit() throws IOException { if (hasErrors) { // 出現I/O錯誤 completeEdit(this, false); remove(entry.key); // the previous entry is stale } else { // 正常情況 completeEdit(this, true); } } // 終止edit數據 public void abort() throws IOException { completeEdit(this, false); } // 封裝的OutputStream,好處是屏蔽掉所有可能出現I/O異常的地方 // 如果出現I/O異常則將hasErrors賦值為false,這樣后面處理邏輯簡單 // 只需要將其理解為不拋出異常而是置位hasErroes標志的OutputStream即可 private final class FaultHidingOutputStream extends FilterOutputStream { private FaultHidingOutputStream(OutputStream out) { super(out); } @Override public void write(int oneByte) { try { out.write(oneByte); } catch (IOException e) { hasErrors = true; } } @Override public void write(byte[] buffer, int offset, int length) { try { out.write(buffer, offset, length); } catch (IOException e) { hasErrors = true; } } @Override public void close() { try { out.close(); } catch (IOException e) { hasErrors = true; } } @Override public void flush() { try { out.flush(); } catch (IOException e) { hasErrors = true; } } } } // LinkedHashMap中的value條目,封裝了一些簡單的信息 private final class Entry { // key,一般是url的MD5 private final String key; // 所對應的文件的大小的數組,如:12876 1567 private final long[] lengths; // 如果該條目被提交過一次即為true private boolean readable; // 該條目所對應的editor,如果正在被edit不為null,否則(已經被寫完了,commited)為null private Editor currentEditor; /** The sequence number of the most recently committed edit to this entry. */ private long sequenceNumber; private Entry(String key) { this.key = key; this.lengths = new long[valueCount]; } // 獲取該條目對應的文件的大小的數組的字符串,用于寫journal public String getLengths() throws IOException { StringBuilder result = new StringBuilder(); for (long size : lengths) { result.append(' ').append(size); } return result.toString(); } //讀取journal文件,解析每行的條目填充該entry private void setLengths(String[] strings) throws IOException { if (strings.length != valueCount) { throw invalidLengths(strings); } try { for (int i = 0; i < strings.length; i++) { lengths[i] = Long.parseLong(strings[i]); } } catch (NumberFormatException e) { throw invalidLengths(strings); } } // 當journal文件的某一行所對應的文件的個數的和valueCount不匹配時調用 private IOException invalidLengths(String[] strings) throws IOException { throw new IOException("unexpected journal line: " + Arrays.toString(strings)); } // 獲取“干凈的”文件:xxx.i public File getCleanFile(int i) { return new File(directory, key + "." + i); } // 獲取“臟的”文件:xxx.i.tmp public File getDirtyFile(int i) { return new File(directory, key + "." + i + ".tmp"); } }
}</pre>