changeset 151:6e14c9e7e53d

Implement efficient Table::get() utilizing index, bloom filters and caching
author Lewin Bormann <lbo@spheniscida.de>
date Sun, 29 Jan 2017 16:14:08 +0100
parents 01e070c97ee7
children ffd4237fd415
files src/blockhandle.rs src/table_builder.rs src/table_reader.rs
diffstat 3 files changed, 52 insertions(+), 31 deletions(-) [+]
line wrap: on
line diff
--- a/src/blockhandle.rs	Sun Jan 29 16:13:36 2017 +0100
+++ b/src/blockhandle.rs	Sun Jan 29 16:14:08 2017 +0100
@@ -4,7 +4,7 @@
 /// used typically as file-internal pointer in table (SSTable) files. For example, the index block
 /// in an SSTable is a block of (key = largest key in block) -> (value = encoded blockhandle of
 /// block).
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct BlockHandle {
     offset: usize,
     size: usize,
--- a/src/table_builder.rs	Sun Jan 29 16:13:36 2017 +0100
+++ b/src/table_builder.rs	Sun Jan 29 16:14:08 2017 +0100
@@ -23,7 +23,7 @@
 pub const TABLE_BLOCK_CKSUM_LEN: usize = 4;
 
 /// Footer is a helper for encoding/decoding a table footer.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Footer {
     pub meta_index: BlockHandle,
     pub index: BlockHandle,
--- a/src/table_reader.rs	Sun Jan 29 16:13:36 2017 +0100
+++ b/src/table_reader.rs	Sun Jan 29 16:14:08 2017 +0100
@@ -77,6 +77,7 @@
     }
 }
 
+#[derive(Clone)]
 pub struct Table<R: Read + Seek> {
     file: R,
     file_size: usize,
@@ -203,38 +204,48 @@
     /// you frequently look for non-existing values (as it will detect the non-existence of an
     /// entry in a block without having to load the block).
     pub fn get<'a>(&mut self, to: InternalKey<'a>) -> Option<Vec<u8>> {
-        let mut iter = self.iter();
+        let mut index_iter = self.indexblock.iter();
+        index_iter.seek(to);
+
+        let handle;
+        if let Some((last_in_block, h)) = index_iter.current() {
+            if self.opt.cmp.cmp(to, &last_in_block) == Ordering::Less {
+                handle = BlockHandle::decode(&h).0;
+            } else {
+                return None;
+            }
+        } else {
+            return None;
+        }
+
+        // found correct block.
 
+        // Check bloom (or whatever) filter
+        if let Some(ref filters) = self.filters {
+            if !filters.key_may_match(handle.offset(), to) {
+                return None;
+            }
+        }
+
+        // Read block (potentially from cache)
+        let mut iter;
+        if let Ok(tb) = self.read_block(&handle) {
+            iter = tb.block.iter();
+        } else {
+            return None;
+        }
+
+        // Go to entry and check if it's the wanted entry.
         iter.seek(to);
-
         if let Some((k, v)) = iter.current() {
-            if k == to { Some(v) } else { None }
+            if self.opt.cmp.cmp(to, &k) == Ordering::Equal {
+                Some(v)
+            } else {
+                None
+            }
         } else {
             None
         }
-
-        // Future impl: TODO
-        //
-        // let mut index_block = self.indexblock.iter();
-        //
-        // index_block.seek(to);
-        //
-        // if let Some((past_block, handle)) = index_block.current() {
-        // if cmp(to, &past_block) == Ordering::Less {
-        // ok, found right block: continue
-        // if let Ok(()) = self.load_block(&handle) {
-        // self.current_block.seek(to);
-        // } else {
-        // return None;
-        // }*/
-        // return None;
-        // } else {
-        // return None;
-        // }
-        // } else {
-        // return None;
-        // }
-        //
     }
 }
 
@@ -534,6 +545,7 @@
                                        size,
                                        BloomPolicy::new(4))
             .unwrap();
+        assert!(table.filters.is_some());
         let filter_reader = table.filters.clone().unwrap();
         let mut iter = table.iter();
 
@@ -649,13 +661,22 @@
                                        size,
                                        BloomPolicy::new(4))
             .unwrap();
+        let mut table2 = table.clone();
+
+        // Test that all of the table's entries are reachable via get()
+        for (k, v) in table.iter() {
+            assert_eq!(table2.get(&k), Some(v));
+        }
+
+        assert_eq!(table.opt.block_cache.lock().unwrap().count(), 3);
 
         assert!(table.get("aaa".as_bytes()).is_none());
-        assert_eq!(table.get("abc".as_bytes()), Some("def".as_bytes().to_vec()));
+        assert!(table.get("aaaa".as_bytes()).is_none());
+        assert!(table.get("aa".as_bytes()).is_none());
         assert!(table.get("abcd".as_bytes()).is_none());
-        assert_eq!(table.get("bcd".as_bytes()), Some("asa".as_bytes().to_vec()));
-        assert_eq!(table.get("zzz".as_bytes()), Some("111".as_bytes().to_vec()));
+        assert!(table.get("zzy".as_bytes()).is_none());
         assert!(table.get("zz1".as_bytes()).is_none());
+        assert!(table.get("zz{".as_bytes()).is_none());
     }
 
     // This test verifies that the table and filters work with internal keys. This means: