changeset 120:28bf33d01950

Build shared block cache into Options
author Lewin Bormann <lbo@spheniscida.de>
date Mon, 26 Dec 2016 11:09:07 +0000
parents b5cdae35b1a4
children 04b1d08a5876
files src/filter.rs src/filter_block.rs src/options.rs src/table_builder.rs
diffstat 4 files changed, 29 insertions(+), 31 deletions(-) [+]
line wrap: on
line diff
--- a/src/filter.rs	Mon Dec 26 10:17:14 2016 +0000
+++ b/src/filter.rs	Mon Dec 26 11:09:07 2016 +0000
@@ -125,14 +125,11 @@
         filter
     }
     fn key_may_match(&self, key: &[u8], filter: &[u8]) -> bool {
-        let len = filter.len();
-
-        if len < 2 {
-            assert!(false);
-            return false;
+        if filter.len() == 0 {
+            return true
         }
 
-        let bits = (len - 1) as u32 * 8;
+        let bits = (filter.len() - 1) as u32 * 8;
         let k = filter[filter.len() - 1];
         let filter_adj = &filter[0..filter.len() - 1];
 
--- a/src/filter_block.rs	Mon Dec 26 10:17:14 2016 +0000
+++ b/src/filter_block.rs	Mon Dec 26 11:09:07 2016 +0000
@@ -144,8 +144,6 @@
             return true;
         }
 
-        println!("{:?}", key);
-
         let filter_begin = self.offset_of(get_filter_index(blk_offset, self.filter_base_lg2));
         let filter_end = self.offset_of(get_filter_index(blk_offset, self.filter_base_lg2) + 1);
 
--- a/src/options.rs	Mon Dec 26 10:17:14 2016 +0000
+++ b/src/options.rs	Mon Dec 26 11:09:07 2016 +0000
@@ -1,6 +1,17 @@
+use block::Block;
+use cache::Cache;
 use types::SequenceNumber;
 
 use std::default::Default;
+use std::sync::Mutex;
+use std::sync::Arc;
+
+const KB: usize = 1 << 10;
+const MB: usize = KB * KB;
+
+const BLOCK_MAX_SIZE: usize = 4 * KB;
+const BLOCK_CACHE_CAPACITY: usize = 8 * MB;
+const WRITE_BUFFER_SIZE: usize = 4 * MB;
 
 #[derive(Clone, Copy, PartialEq, Debug)]
 pub enum CompressionType {
@@ -18,7 +29,7 @@
 
 /// [not all member types implemented yet]
 ///
-#[derive(Clone, Copy)]
+#[derive(Clone)]
 pub struct Options {
     pub create_if_missing: bool,
     pub error_if_exists: bool,
@@ -26,7 +37,7 @@
     // pub logger: Logger,
     pub write_buffer_size: usize,
     pub max_open_files: usize,
-    // pub block_cache: Cache,
+    pub block_cache: Arc<Mutex<Cache<Block>>>,
     pub block_size: usize,
     pub block_restart_interval: usize,
     pub compression_type: CompressionType,
@@ -39,9 +50,10 @@
             create_if_missing: true,
             error_if_exists: false,
             paranoid_checks: false,
-            write_buffer_size: 4 * (1 << 20),
+            write_buffer_size: WRITE_BUFFER_SIZE,
             max_open_files: 1 << 10,
-            block_size: 4 * (1 << 10),
+            block_cache: Arc::new(Mutex::new(Cache::new(BLOCK_CACHE_CAPACITY / BLOCK_MAX_SIZE))), /* 2000 elements */
+            block_size: BLOCK_MAX_SIZE,
             block_restart_interval: 16,
             reuse_logs: false,
             compression_type: CompressionType::CompressionNone,
--- a/src/table_builder.rs	Mon Dec 26 10:17:14 2016 +0000
+++ b/src/table_builder.rs	Mon Dec 26 11:09:07 2016 +0000
@@ -101,7 +101,7 @@
 /// 0xdb4775248b80fb57.
 
 pub struct TableBuilder<'a, Dst: Write, FilterPol: FilterPolicy> {
-    o: Options,
+    opt: Options,
     dst: Dst,
 
     offset: usize,
@@ -115,16 +115,7 @@
 
 impl<'a, Dst: Write> TableBuilder<'a, Dst, NoFilterPolicy> {
     pub fn new_no_filter(opt: Options, dst: Dst) -> TableBuilder<'a, Dst, NoFilterPolicy> {
-        TableBuilder {
-            o: opt,
-            dst: dst,
-            offset: 0,
-            prev_block_last_key: vec![],
-            num_entries: 0,
-            data_block: Some(BlockBuilder::new(opt)),
-            index_block: Some(BlockBuilder::new(opt)),
-            filter_block: None,
-        }
+        TableBuilder::new(opt, dst, NoFilterPolicy)
     }
 }
 
@@ -135,12 +126,12 @@
 impl<'a, Dst: Write, FilterPol: FilterPolicy> TableBuilder<'a, Dst, FilterPol> {
     pub fn new(opt: Options, dst: Dst, fpol: FilterPol) -> TableBuilder<'a, Dst, FilterPol> {
         TableBuilder {
-            o: opt,
+            opt: opt.clone(),
             dst: dst,
             offset: 0,
             prev_block_last_key: vec![],
             num_entries: 0,
-            data_block: Some(BlockBuilder::new(opt)),
+            data_block: Some(BlockBuilder::new(opt.clone())),
             index_block: Some(BlockBuilder::new(opt)),
             filter_block: Some(FilterBlockBuilder::new(fpol)),
         }
@@ -154,7 +145,7 @@
         assert!(self.data_block.is_some());
         assert!(self.num_entries == 0 || cmp(&self.prev_block_last_key, key) == Ordering::Less);
 
-        if self.data_block.as_ref().unwrap().size_estimate() > self.o.block_size {
+        if self.data_block.as_ref().unwrap().size_estimate() > self.opt.block_size {
             self.write_data_block(key);
         }
 
@@ -184,9 +175,9 @@
         let enc_len = handle.encode_to(&mut handle_enc);
 
         self.index_block.as_mut().unwrap().add(&sep, &handle_enc[0..enc_len]);
-        self.data_block = Some(BlockBuilder::new(self.o));
+        self.data_block = Some(BlockBuilder::new(self.opt.clone()));
 
-        let ctype = self.o.compression_type;
+        let ctype = self.opt.compression_type;
 
         self.write_block(contents, ctype);
 
@@ -204,7 +195,7 @@
         let mut digest = crc32::Digest::new(crc32::CASTAGNOLI);
 
         digest.write(&block);
-        digest.write(&[self.o.compression_type as u8; TABLE_BLOCK_COMPRESS_LEN]);
+        digest.write(&[self.opt.compression_type as u8; TABLE_BLOCK_COMPRESS_LEN]);
         digest.sum32().encode_fixed(&mut buf);
 
         // TODO: Handle errors here.
@@ -221,7 +212,7 @@
 
     pub fn finish(mut self) {
         assert!(self.data_block.is_some());
-        let ctype = self.o.compression_type;
+        let ctype = self.opt.compression_type;
 
         // If there's a pending data block, write it
         if self.data_block.as_ref().unwrap().entries() > 0 {
@@ -229,7 +220,7 @@
         }
 
         // Create metaindex block
-        let mut meta_ix_block = BlockBuilder::new(self.o);
+        let mut meta_ix_block = BlockBuilder::new(self.opt.clone());
 
         if self.filter_block.is_some() {
             // if there's a filter block, write the filter block and add it to the metaindex block.