changeset 116:02b8cf7b62e5

Prepare for filter policies in TableReader; add general filter tests
author Lewin Bormann <lbo@spheniscida.de>
date Sun, 25 Dec 2016 09:13:43 +0000
parents 48c612aff71c
children 6147b3a3eeea
files src/filter_block.rs src/options.rs src/table_builder.rs src/table_reader.rs
diffstat 4 files changed, 41 insertions(+), 9 deletions(-) [+]
line wrap: on
line diff
--- a/src/filter_block.rs	Sat Dec 24 20:58:35 2016 +0000
+++ b/src/filter_block.rs	Sun Dec 25 09:13:43 2016 +0000
@@ -98,6 +98,7 @@
     }
 }
 
+#[derive(Clone)]
 pub struct FilterBlockReader<FP: FilterPolicy> {
     policy: FP,
     block: Rc<Vec<u8>>,
@@ -126,7 +127,7 @@
     }
 
     /// Returns number of filters
-    fn num(&self) -> u32 {
+    pub fn num(&self) -> u32 {
         ((self.block.len() - self.offsets_offset - 5) / 4) as u32
     }
 
@@ -140,7 +141,7 @@
     /// filter for the block at blk_offset.
     pub fn key_may_match(&self, blk_offset: usize, key: &[u8]) -> bool {
         if get_filter_index(blk_offset, self.filter_base_lg2) > self.num() {
-            return true
+            return true;
         }
 
         let filter_begin = self.offset_of(get_filter_index(blk_offset, self.filter_base_lg2));
--- a/src/options.rs	Sat Dec 24 20:58:35 2016 +0000
+++ b/src/options.rs	Sun Dec 25 09:13:43 2016 +0000
@@ -12,7 +12,7 @@
     match i {
         0 => Some(CompressionType::CompressionNone),
         1 => Some(CompressionType::CompressionSnappy),
-        _ => None
+        _ => None,
     }
 }
 
--- a/src/table_builder.rs	Sat Dec 24 20:58:35 2016 +0000
+++ b/src/table_builder.rs	Sun Dec 25 09:13:43 2016 +0000
@@ -237,9 +237,8 @@
         assert!(self.data_block.is_some());
         let ctype = self.o.compression_type;
 
-        // If there's a pending data block, write that one
-        let flush_last_block = self.data_block.as_ref().unwrap().entries() > 0;
-        if flush_last_block {
+        // If there's a pending data block, write it
+        if self.data_block.as_ref().unwrap().entries() > 0 {
             self.write_data_block(&[0xff as u8; 1]);
         }
 
@@ -329,6 +328,7 @@
             b.add(k.as_bytes(), v.as_bytes());
         }
 
+        assert!(b.filter_block.is_some());
         b.finish();
     }
 
--- a/src/table_reader.rs	Sat Dec 24 20:58:35 2016 +0000
+++ b/src/table_reader.rs	Sun Dec 25 09:13:43 2016 +0000
@@ -96,9 +96,9 @@
                                       "Indexblock/Metaindexblock failed verification"));
         }
 
+        // Open filter block for reading
         let mut filter_block_reader = None;
-        let mut filter_name = "filter.".as_bytes().to_vec();
-        filter_name.extend_from_slice(fp.name().as_bytes());
+        let filter_name = format!("filter.{}", fp.name()).as_bytes().to_vec();
 
         let mut metaindexiter = metaindexblock.block.iter();
 
@@ -154,6 +154,7 @@
     fn iter<'a>(&'a mut self) -> TableIterator<'a, R, C, FP> {
         let iter = TableIterator {
             current_block: self.indexblock.iter(), // just for filling in here
+            current_block_off: 0,
             index_block: self.indexblock.iter(),
             table: self,
             init: false,
@@ -180,6 +181,7 @@
 pub struct TableIterator<'a, R: 'a + Read + Seek, C: 'a + Comparator, FP: 'a + FilterPolicy> {
     table: &'a mut Table<R, C, FP>,
     current_block: BlockIter<C>,
+    current_block_off: usize,
     index_block: BlockIter<C>,
 
     init: bool,
@@ -202,6 +204,7 @@
 
         let block = try!(self.table.read_block(&new_block_handle));
         self.current_block = block.block.iter();
+        self.current_block_off = new_block_handle.offset();
 
         Ok(())
     }
@@ -342,6 +345,7 @@
             }
 
             b.finish();
+
         }
 
         let size = d.len();
@@ -363,6 +367,9 @@
                                    Options::default())
             .unwrap();
 
+        assert!(table.filters.is_some());
+        assert_eq!(table.filters.as_ref().unwrap().num(), 1);
+
         {
             let iter = table.iter();
             // Last block is skipped
@@ -404,7 +411,31 @@
         }
 
         assert_eq!(i, data.len());
-        println!("tot len: {}", data.len());
+    }
+
+    #[test]
+    fn test_table_iterator_filter() {
+        let (src, size) = build_table();
+        let data = build_data();
+
+        let mut table = Table::new(Cursor::new(&src as &[u8]),
+                                   size,
+                                   StandardComparator,
+                                   BloomPolicy::new(4),
+                                   Options::default())
+            .unwrap();
+        let filter_reader = table.filters.clone().unwrap();
+        let mut iter = table.iter();
+
+        loop {
+            if let Some((k, _)) = iter.next() {
+                assert!(filter_reader.key_may_match(iter.current_block_off, &k));
+                assert!(!filter_reader.key_may_match(iter.current_block_off,
+                                                     "somerandomkey".as_bytes()));
+            } else {
+                break;
+            }
+        }
     }
 
     #[test]