changeset 129:290af663edcf

Extend read_block() logic to also read the block checksum
author Lewin Bormann <lbo@spheniscida.de>
date Mon, 19 Dec 2016 19:40:25 +0100
parents c48c912aee64
children ac1a13a5edef
files src/env.rs src/table_builder.rs src/table_reader.rs src/version_edit.rs
diffstat 4 files changed, 41 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- a/src/env.rs	Sun Nov 13 14:55:00 2016 +0100
+++ b/src/env.rs	Mon Dec 19 19:40:25 2016 +0100
@@ -47,3 +47,15 @@
         let _ = self.dst.write("\n".as_bytes());
     }
 }
+
+fn make_file_name(db: &str, num: u64, ext: &str) -> String {
+    format!("{}/{:06}.{}", db, num, ext)
+}
+
+pub fn table_file_name(db: &str, num: u64) -> String {
+    make_file_name(db, num, "ldb")
+}
+
+pub fn log_file_name(db: &str, num: u64) -> String {
+    make_file_name(db, num, "log")
+}
--- a/src/table_builder.rs	Sun Nov 13 14:55:00 2016 +0100
+++ b/src/table_builder.rs	Mon Dec 19 19:40:25 2016 +0100
@@ -13,6 +13,7 @@
 use crc::Hasher32;
 use integer_encoding::FixedInt;
 
+pub const TABLE_BLOCK_CHECKSUM_LENGTH: usize = 5;
 pub const FOOTER_LENGTH: usize = 40;
 pub const FULL_FOOTER_LENGTH: usize = FOOTER_LENGTH + 8;
 pub const MAGIC_FOOTER_NUMBER: u64 = 0xdb4775248b80fb57;
@@ -91,15 +92,18 @@
     }
 }
 
-/// A table consists of DATA BLOCKs, META BLOCKs, a METAINDEX BLOCK, an INDEX BLOCK and a FOOTER.
+/// A table consists of DATA BLOCKs, an INDEX BLOCK and a FOOTER.
 ///
-/// DATA BLOCKs, META BLOCKs, INDEX BLOCK and METAINDEX BLOCK are built using the code in
-/// the `block` module.
+/// DATA BLOCKs, INDEX BLOCKs, and BLOCKs are built using the code in the `block` module.
 ///
-/// The FOOTER consists of a BlockHandle wthat points to the metaindex block, another pointing to
-/// the index block, padding to fill up to 40 B and at the end the 8B magic number
-/// 0xdb4775248b80fb57.
-
+/// DATA BLOCKs contain the actual data, and a footer of [1B compression; 4B crc32];
+/// INDEX BLOCKS contain one entry per block, where the key is a string after the
+/// last key of a block, and the value is a encoded BlockHandle pointing to that
+/// block.
+///
+/// The footer is a pointer pointing to the index block, padding to fill up to 40 B and at the end
+/// the 8B magic number 0xdb4775248b80fb57.
+///
 pub struct TableBuilder<'a, C: Comparator, Dst: Write, FilterPol: FilterPolicy> {
     o: Options,
     cmp: C,
--- a/src/table_reader.rs	Sun Nov 13 14:55:00 2016 +0100
+++ b/src/table_reader.rs	Mon Dec 19 19:40:25 2016 +0100
@@ -9,6 +9,10 @@
 use std::io::{Read, Seek, SeekFrom, Result};
 use std::cmp::Ordering;
 
+use integer_encoding::FixedInt;
+
+type BlockChecksum = u32;
+
 /// Reads the table footer.
 fn read_footer<R: Read + Seek>(f: &mut R, size: usize) -> Result<Footer> {
     try!(f.seek(SeekFrom::Start((size - table_builder::FULL_FOOTER_LENGTH) as u64)));
@@ -28,13 +32,20 @@
     Ok(buf)
 }
 
-/// Reads a block at location.
+/// Reads a table block and its checksum at location.
 fn read_block<R: Read + Seek, C: Comparator>(cmp: &C,
                                              f: &mut R,
                                              location: &BlockHandle)
-                                             -> Result<Block<C>> {
+                                             -> Result<(Block<C>, BlockChecksum)> {
     let buf = try!(read_bytes(f, location));
-    Ok(Block::new(buf, *cmp))
+
+    let cksum_location = BlockHandle::new(location.offset() + location.size(),
+                                          table_builder::TABLE_BLOCK_CHECKSUM_LENGTH);
+    let cksum_buf = try!(read_bytes(f, &cksum_location));
+
+    let cksum = u32::decode_fixed(&cksum_buf[1..]);
+
+    Ok((Block::new(buf, *cmp), cksum))
 }
 
 pub struct Table<R: Read + Seek, C: Comparator, FP: FilterPolicy> {
@@ -53,8 +64,8 @@
     pub fn new(mut file: R, size: usize, cmp: C, fp: FP, opt: Options) -> Result<Table<R, C, FP>> {
         let footer = try!(read_footer(&mut file, size));
 
-        let indexblock = try!(read_block(&cmp, &mut file, &footer.index));
-        let metaindexblock = try!(read_block(&cmp, &mut file, &footer.meta_index));
+        let (indexblock, _) = try!(read_block(&cmp, &mut file, &footer.index));
+        let (metaindexblock, _) = try!(read_block(&cmp, &mut file, &footer.meta_index));
 
         let mut filter_block_reader = None;
         let mut filter_name = "filter.".as_bytes().to_vec();
@@ -86,7 +97,7 @@
     }
 
     fn read_block_(&mut self, location: &BlockHandle) -> Result<Block<C>> {
-        read_block(&self.cmp, &mut self.file, location)
+        read_block(&self.cmp, &mut self.file, location).map(|t| t.0)
     }
 
     /// Returns the offset of the block that contains `key`.
--- a/src/version_edit.rs	Sun Nov 13 14:55:00 2016 +0100
+++ b/src/version_edit.rs	Mon Dec 19 19:40:25 2016 +0100
@@ -60,6 +60,7 @@
 }
 
 /// Manages changes to the set of managed SSTables and logfiles.
+#[derive(Debug)]
 pub struct VersionEdit {
     comparator: Option<String>,
     log_number: Option<u64>,