Mercurial > lbo > hg > sstable
changeset 43:08b29b687320
More cleanups of unused code.
| author | Lewin Bormann <lbo@spheniscida.de> |
|---|---|
| date | Sun, 11 Mar 2018 16:12:39 +0100 |
| parents | 9cf43f11d5a6 |
| children | d0d878a8cbcc |
| files | src/block.rs src/filter_block.rs src/iterator.rs src/lib.rs src/options.rs src/table_builder.rs src/table_reader.rs src/types.rs |
| diffstat | 8 files changed, 55 insertions(+), 239 deletions(-) [+] |
line wrap: on
line diff
--- a/src/block.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/block.rs Sun Mar 11 16:12:39 2018 +0100 @@ -324,7 +324,7 @@ #[test] fn test_block_iterator_properties() { - let o = options::for_test(); + let o = Options::default(); let mut builder = BlockBuilder::new(o.clone()); let mut data = get_data(); data.truncate(4); @@ -339,7 +339,7 @@ #[test] fn test_block_empty() { - let mut o = options::for_test(); + let mut o = Options::default(); o.block_restart_interval = 16; let builder = BlockBuilder::new(o); @@ -347,7 +347,7 @@ assert_eq!(blockc.len(), 8); assert_eq!(blockc, vec![0, 0, 0, 0, 1, 0, 0, 0]); - let block = Block::new(options::for_test(), blockc); + let block = Block::new(Options::default(), blockc); for _ in SSIteratorIter::wrap(&mut block.iter()) { panic!("expected 0 iterations"); @@ -357,14 +357,14 @@ #[test] fn test_block_build_iterate() { let data = get_data(); - let mut builder = BlockBuilder::new(options::for_test()); + let mut builder = BlockBuilder::new(Options::default()); for &(k, v) in data.iter() { builder.add(k, v); } let block_contents = builder.finish(); - let mut block = Block::new(options::for_test(), block_contents).iter(); + let mut block = Block::new(Options::default(), block_contents).iter(); let mut i = 0; assert!(!block.valid()); @@ -379,7 +379,7 @@ #[test] fn test_block_iterate_reverse() { - let mut o = options::for_test(); + let mut o = Options::default(); o.block_restart_interval = 3; let data = get_data(); let mut builder = BlockBuilder::new(o.clone()); @@ -425,7 +425,7 @@ #[test] fn test_block_seek() { - let mut o = options::for_test(); + let mut o = Options::default(); o.block_restart_interval = 3; let data = get_data(); @@ -483,7 +483,7 @@ #[test] fn test_block_seek_to_last() { - let mut o = options::for_test(); + let mut o = Options::default(); // Test with different number of restarts for block_restart_interval in vec![2, 6, 10] {
--- a/src/filter_block.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/filter_block.rs Sun Mar 11 16:12:39 2018 +0100 @@ -6,7 +6,6 @@ use integer_encoding::FixedInt; const FILTER_BASE_LOG2: u32 = 11; -const FILTER_BASE: u32 = 1 << FILTER_BASE_LOG2; // 2 KiB /// For a given byte offset, returns the index of the filter that includes the key at that offset. #[inline]
--- a/src/iterator.rs Sun Mar 11 16:00:45 2018 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,46 +0,0 @@ -//! A collection of fundamental and/or simple types used by other modules - -use std::cmp::Ordering; - -/// Trait used to influence how SkipMap determines the order of elements. Use StandardComparator -/// for the normal implementation using numerical comparison. -pub trait Comparator: Copy { - fn cmp(&self, &[u8], &[u8]) -> Ordering; -} - -#[derive(Clone, Copy, Default)] -pub struct StandardComparator; - -impl Comparator for StandardComparator { - fn cmp(&self, a: &[u8], b: &[u8]) -> Ordering { - a.cmp(b) - } -} - -/// An extension of the standard `Iterator` trait that supports some methods necessary for LevelDB. -/// This works because the iterators used are stateful and keep the last returned element. -/// -/// Note: Implementing types are expected to hold `!valid()` before the first call to `next()`. -pub trait SSIterator: Iterator { - // We're emulating LevelDB's Slice type here using actual slices with the lifetime of the - // iterator. The lifetime of the iterator is usually the one of the backing storage (Block, - // MemTable, SkipMap...) - // type Item = (&'a [u8], &'a [u8]); - - /// Seek the iterator to `key` or the next bigger key. If the seek is invalid (past last - /// element), the iterator is reset() and not valid. - fn seek(&mut self, key: &[u8]); - /// Resets the iterator to be `!valid()` again (before first element) - fn reset(&mut self); - /// Returns true if `current()` would return a valid item. - fn valid(&self) -> bool; - /// Return the current item. - fn current(&self) -> Option<Self::Item>; - /// Go to the previous item. - fn prev(&mut self) -> Option<Self::Item>; - - fn seek_to_first(&mut self) { - self.reset(); - self.next(); - } -}
--- a/src/lib.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/lib.rs Sun Mar 11 16:12:39 2018 +0100 @@ -13,19 +13,18 @@ pub mod filter; mod filter_block; mod table_block; -mod test_util; mod types; -pub mod cmp; -pub mod iterator; -pub mod options; -pub mod table_builder; -pub mod table_reader; +mod cmp; +mod options; +mod table_builder; +mod table_reader; -pub use cmp::Cmp; -pub use iterator::StandardComparator; -pub use iterator::SSIterator; +pub use cmp::{Cmp, DefaultCmp}; pub use options::Options; - +pub use types::{current_key_val, SSIterator}; pub use table_builder::TableBuilder; pub use table_reader::{Table, TableIterator}; + +#[cfg(test)] +mod test_util;
--- a/src/options.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/options.rs Sun Mar 11 16:12:39 2018 +0100 @@ -58,7 +58,3 @@ } } } - -pub fn for_test() -> Options { - Options::default() -}
--- a/src/table_builder.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/table_builder.rs Sun Mar 11 16:12:39 2018 +0100 @@ -5,7 +5,7 @@ use filter::NoFilterPolicy; use filter_block::FilterBlockBuilder; use options::{CompressionType, Options}; -use types::{mask_crc, unmask_crc}; +use types::mask_crc; use std::cmp::Ordering; use std::io::Write; @@ -18,8 +18,7 @@ pub const FOOTER_LENGTH: usize = 40; pub const FULL_FOOTER_LENGTH: usize = FOOTER_LENGTH + 8; -pub const MAGIC_FOOTER_NUMBER: u64 = 0xdb4775248b80fb57; -pub const MAGIC_FOOTER_ENCODED: [u8; 8] = [0x57, 0xfb, 0x80, 0x8b, 0x24, 0x75, 0x47, 0xdb]; +const MAGIC_FOOTER_ENCODED: [u8; 8] = [0x57, 0xfb, 0x80, 0x8b, 0x24, 0x75, 0x47, 0xdb]; pub const TABLE_BLOCK_COMPRESS_LEN: usize = 1; pub const TABLE_BLOCK_CKSUM_LEN: usize = 4; @@ -70,15 +69,17 @@ } } -/// A table consists of DATA BLOCKs, META BLOCKs, a METAINDEX BLOCK, an INDEX BLOCK and a FOOTER. -/// -/// DATA BLOCKs, META BLOCKs, INDEX BLOCK and METAINDEX BLOCK are built using the code in -/// the `block` module. -/// -/// The FOOTER consists of a BlockHandle that points to the metaindex block, another pointing to -/// the index block, padding to fill up to 40 B and at the end the 8B magic number -/// 0xdb4775248b80fb57. +/// A TableBuilder is used to create a table from a set of sorted string pairs and write it to a +/// file or a buffer. +// A table consists of DATA BLOCKs, META BLOCKs, a METAINDEX BLOCK, an INDEX BLOCK and a FOOTER. +// +// DATA BLOCKs, META BLOCKs, INDEX BLOCK and METAINDEX BLOCK are built using the code in +// the `block` module. +// +// The FOOTER consists of a BlockHandle that points to the metaindex block, another pointing to +// the index block, padding to fill up to 40 B and at the end the 8B magic number +// 0xdb4775248b80fb57. pub struct TableBuilder<Dst: Write> { opt: Options, dst: Dst, @@ -116,11 +117,12 @@ } } + /// Returns the current number of entries. pub fn entries(&self) -> usize { self.num_entries } - pub fn size_estimate(&self) -> usize { + fn size_estimate(&self) -> usize { let mut size = 0; if let Some(ref b) = self.data_block { size += b.size_estimate(); @@ -134,7 +136,8 @@ size + self.offset + FULL_FOOTER_LENGTH } - /// Add a key to the table. The key as to be lexically greater or equal to the last one added. + /// Add a key to the table. The key must be lexically greater or equal to the one that was + /// previously added. pub fn add(&mut self, key: &[u8], val: &[u8]) -> Result<()> { assert!(self.data_block.is_some()); @@ -280,7 +283,7 @@ #[test] fn test_table_builder() { let mut d = Vec::with_capacity(512); - let mut opt = options::for_test(); + let mut opt = Options::default(); opt.block_restart_interval = 3; opt.compression_type = CompressionType::CompressionSnappy; let mut b = TableBuilder::new(opt, &mut d); @@ -317,7 +320,7 @@ #[should_panic] fn test_bad_input() { let mut d = Vec::with_capacity(512); - let mut opt = options::for_test(); + let mut opt = Options::default(); opt.block_restart_interval = 3; let mut b = TableBuilder::new(opt, &mut d);
--- a/src/table_reader.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/table_reader.rs Sun Mar 11 16:12:39 2018 +0100 @@ -2,7 +2,6 @@ use blockhandle::BlockHandle; use cache; use error::Result; -use filter; use filter_block::FilterBlockReader; use options::Options; use table_block; @@ -347,7 +346,6 @@ #[cfg(test)] mod tests { - use filter::BloomPolicy; use options::{self, CompressionType}; use table_builder::TableBuilder; use test_util::{test_iterator_properties, SSIteratorIter}; @@ -374,7 +372,7 @@ // reason, a call f(v, v.len()) doesn't work for borrowing reasons. fn build_table(data: Vec<(&'static str, &'static str)>) -> (Vec<u8>, usize) { let mut d = Vec::with_capacity(512); - let mut opt = options::for_test(); + let mut opt = Options::default(); opt.block_restart_interval = 2; opt.block_size = 32; opt.compression_type = CompressionType::CompressionSnappy; @@ -394,32 +392,6 @@ (d, size) } - // Build a table containing keys. - fn build_internal_table() -> (Vec<u8>, usize) { - let mut d = Vec::with_capacity(512); - let mut opt = options::for_test(); - opt.block_restart_interval = 1; - opt.block_size = 32; - opt.filter_policy = Rc::new(Box::new(BloomPolicy::new(4))); - - let mut i = 1 as u64; - let data = build_data(); - - { - let mut b = TableBuilder::new(opt, &mut d); - - for &(ref k, ref v) in data.iter() { - b.add(k.as_bytes(), v.as_bytes()).unwrap(); - } - - b.finish().unwrap(); - } - - let size = d.len(); - - (d, size) - } - fn wrap_buffer(src: Vec<u8>) -> Rc<Box<RandomAccess>> { Rc::new(Box::new(src)) } @@ -427,7 +399,7 @@ #[test] fn test_table_approximate_offset() { let (src, size) = build_table(build_data()); - let mut opt = options::for_test(); + let mut opt = Options::default(); opt.block_size = 32; let table = Table::new(opt, wrap_buffer(src), size).unwrap(); let mut iter = table.iter(); @@ -446,7 +418,7 @@ #[test] fn test_table_block_cache_use() { let (src, size) = build_table(build_data()); - let mut opt = options::for_test(); + let mut opt = Options::default(); opt.block_size = 32; let table = Table::new(opt.clone(), wrap_buffer(src), size).unwrap(); @@ -469,7 +441,7 @@ let (src, size) = build_table(build_data()); let data = build_data(); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); let mut iter = table.iter(); let mut i = 0; @@ -519,7 +491,7 @@ fn test_table_iterator_filter() { let (src, size) = build_table(build_data()); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); assert!(table.filters.is_some()); let filter_reader = table.filters.clone().unwrap(); let mut iter = table.iter(); @@ -538,7 +510,7 @@ fn test_table_iterator_state_behavior() { let (src, size) = build_table(build_data()); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); let mut iter = table.iter(); // behavior test @@ -568,7 +540,7 @@ let mut data = build_data(); data.truncate(4); let (src, size) = build_table(data); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); test_iterator_properties(table.iter()); } @@ -577,7 +549,7 @@ let (src, size) = build_table(build_data()); let data = build_data(); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); let mut iter = table.iter(); let mut i = 0; @@ -612,7 +584,7 @@ fn test_table_iterator_seek() { let (src, size) = build_table(build_data()); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); let mut iter = table.iter(); iter.seek(b"bcd"); @@ -639,7 +611,7 @@ fn test_table_get() { let (src, size) = build_table(build_data()); - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); let table2 = table.clone(); let mut _iter = table.iter(); @@ -668,7 +640,7 @@ src[10] += 1; - let table = Table::new(options::for_test(), wrap_buffer(src), size).unwrap(); + let table = Table::new(Options::default(), wrap_buffer(src), size).unwrap(); assert!(table.filters.is_some()); assert_eq!(table.filters.as_ref().unwrap().num(), 1);
--- a/src/types.rs Sun Mar 11 16:00:45 2018 +0100 +++ b/src/types.rs Sun Mar 11 16:12:39 2018 +0100 @@ -1,6 +1,6 @@ //! A collection of fundamental and/or simple types used by other modules. A bit of a grab bag :-) -use error::{err, Result, StatusCode}; +use error::Result; use std::cell::RefCell; use std::rc::Rc; @@ -9,7 +9,8 @@ fn read_at(&self, off: usize, dst: &mut [u8]) -> Result<usize>; } -/// BufferBackedFile is a simple type implementing RandomAccess on a Vec<u8>. +/// BufferBackedFile is a simple type implementing RandomAccess on a Vec<u8>. Used for some tests. +#[allow(unused)] pub type BufferBackedFile = Vec<u8>; impl RandomAccess for BufferBackedFile { @@ -28,13 +29,6 @@ } } -pub const NUM_LEVELS: usize = 7; - -/// Represents a sequence number of a single entry. -pub type SequenceNumber = u64; - -pub const MAX_SEQUENCE_NUMBER: SequenceNumber = (1 << 56) - 1; - /// A shared thingy with interior mutability. pub type Shared<T> = Rc<RefCell<T>>; @@ -42,24 +36,13 @@ Rc::new(RefCell::new(t)) } -#[derive(PartialEq)] -pub enum Direction { - Forward, - Reverse, -} - -/// Denotes a key range -pub struct Range<'a> { - pub start: &'a [u8], - pub limit: &'a [u8], -} - -/// An extension of the standard `Iterator` trait that supports some methods necessary for LevelDB. -/// This works because the iterators used are stateful and keep the last returned element. +/// An extension of the standard `Iterator` trait that supporting some additional functionality. /// -/// Note: Implementing types are expected to hold `!valid()` before the first call to `advance()`. +/// Note: Implementing types are expected to hold `!valid()` before the first call to `advance()`, +/// and after `advance()` has returned `false` for the first time. /// -/// test_util::test_iterator_properties() verifies that all properties hold. +/// test_util::test_iterator_properties() verifies that all properties hold for a given +/// implementation. pub trait SSIterator { /// Advances the position of the iterator by one element (which can be retrieved using /// current(). If no more elements are available, advance() returns false, and the iterator @@ -136,74 +119,6 @@ } } -/// The unique (sequential) number of a file. -pub type FileNum = u64; - -/// Describes a file on disk. -#[derive(Clone, Debug, Default, PartialEq)] -pub struct FileMetaData { - // default: size / 16384. - pub allowed_seeks: usize, - pub num: FileNum, - pub size: usize, - // these are in InternalKey format: - pub smallest: Vec<u8>, - pub largest: Vec<u8>, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum FileType { - Log, - DBLock, - Table, - Descriptor, - Current, - Temp, - InfoLog, -} - -pub fn parse_file_name(f: &str) -> Result<(FileNum, FileType)> { - if f == "CURRENT" { - return Ok((0, FileType::Current)); - } else if f == "LOCK" { - return Ok((0, FileType::DBLock)); - } else if f == "LOG" || f == "LOG.old" { - return Ok((0, FileType::InfoLog)); - } else if f.starts_with("MANIFEST-") { - if let Some(ix) = f.find('-') { - if let Ok(num) = FileNum::from_str_radix(&f[ix + 1..], 10) { - return Ok((num, FileType::Descriptor)); - } - return err( - StatusCode::InvalidArgument, - "manifest file number is invalid", - ); - } - return err(StatusCode::InvalidArgument, "manifest file has no dash"); - } else if let Some(ix) = f.find('.') { - // 00012345.log 00123.sst ... - if let Ok(num) = FileNum::from_str_radix(&f[0..ix], 10) { - let typ = match &f[ix + 1..] { - "log" => FileType::Log, - "sst" | "ldb" => FileType::Table, - "dbtmp" => FileType::Temp, - _ => { - return err( - StatusCode::InvalidArgument, - "unknown numbered file extension", - ) - } - }; - return Ok((num, typ)); - } - return err( - StatusCode::InvalidArgument, - "invalid file number for table or temp file", - ); - } - err(StatusCode::InvalidArgument, "unknown file type") -} - const MASK_DELTA: u32 = 0xa282ead8; pub fn mask_crc(c: u32) -> u32 { @@ -216,26 +131,4 @@ } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_types_parse_file_name() { - for c in &[ - ("CURRENT", (0, FileType::Current)), - ("LOCK", (0, FileType::DBLock)), - ("LOG", (0, FileType::InfoLog)), - ("LOG.old", (0, FileType::InfoLog)), - ("MANIFEST-01234", (1234, FileType::Descriptor)), - ("001122.sst", (1122, FileType::Table)), - ("001122.ldb", (1122, FileType::Table)), - ("001122.dbtmp", (1122, FileType::Temp)), - ] { - assert_eq!(parse_file_name(c.0).unwrap(), c.1); - } - assert!(parse_file_name("xyz.LOCK").is_err()); - assert!(parse_file_name("01a.sst").is_err()); - assert!(parse_file_name("0011.abc").is_err()); - assert!(parse_file_name("MANIFEST-trolol").is_err()); - } -} +mod tests {}