Mercurial > lbo > hg > leveldb-rs

changeset 610:9a561cd122c3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add CompressorList & Refactor
author kaiyohugo <41114603+KAIYOHUGO@users.noreply.github.com>
date Sat, 15 Jul 2023 14:05:55 +0800
parents 68887f0a3e83
children ca88834b15c2
files examples/leveldb-tool/src/main.rs examples/stresstest/src/main.rs src/compressor.rs src/db_impl.rs src/error.rs src/lib.rs src/options.rs src/table_block.rs src/table_builder.rs
diffstat 9 files changed, 111 insertions(+), 49 deletions(-) [+]
line wrap: on
line diff
--- a/examples/leveldb-tool/src/main.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/examples/leveldb-tool/src/main.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -1,6 +1,6 @@
 extern crate rusty_leveldb;
 
-use rusty_leveldb::{CompressorId, LdbIterator, Options, DB};
+use rusty_leveldb::{compressor, CompressorId, LdbIterator, Options, DB};
 
 use std::env::args;
 use std::io::{self, Write};
@@ -59,7 +59,7 @@
     let mut opt = Options::default();
     opt.reuse_logs = false;
     opt.reuse_manifest = false;
-    opt.compressor = rusty_leveldb::compressor::SnappyCompressor::ID;
+    opt.compressor = compressor::SnappyCompressor::ID;
     let mut db = DB::open("tooldb", opt).unwrap();
 
     match args[1].as_str() {
--- a/examples/stresstest/src/main.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/examples/stresstest/src/main.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -12,7 +12,7 @@
 
 fn write(db: &mut DB, n: usize) {
     time_test::time_test!("write");
-    for i in 0..n {
+    for _ in 0..n {
         let (k, v) = (gen_string(KEY_LEN), gen_string(VAL_LEN));
 
         db.put(k.as_bytes(), v.as_bytes()).unwrap();
@@ -27,10 +27,10 @@
 fn read(db: &mut DB, n: usize) -> usize {
     let mut succ = 0;
     time_test::time_test!("read");
-    for i in 0..n {
+    for _ in 0..n {
         let k = gen_string(KEY_LEN);
 
-        if let Some(v) = db.get(k.as_bytes()) {
+        if let Some(_) = db.get(k.as_bytes()) {
             succ += 1;
         }
     }
@@ -38,7 +38,7 @@
 }
 
 fn main() {
-    let N = 100_000;
+    let n = 100_000;
     let m = 10;
     let path = "stresstestdb";
     let mut entries = 0;
@@ -47,11 +47,11 @@
         let mut opt = Options::default();
         opt.compressor = compressor::SnappyCompressor::ID;
         let mut db = DB::open(path, opt).unwrap();
-        write(&mut db, N);
-        entries += N;
+        write(&mut db, n);
+        entries += n;
         println!("Wrote {} entries ({}/{})", entries, i + 1, m);
 
-        let s = read(&mut db, N);
-        println!("Read back {} entries (found {}) ({}/{})", N, s, i + 1, m);
+        let s = read(&mut db, n);
+        println!("Read back {} entries (found {}) ({}/{})", n, s, i + 1, m);
     }
 }
--- a/src/compressor.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/compressor.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -1,22 +1,46 @@
+/// Custom compression method
+///
+/// ```
+/// # use rusty_leveldb::{Compressor, CompressorId};
+/// 
+/// #[derive(Debug, Clone, Copy, Default)]
+/// pub struct CustomCompressor;
+///
+/// impl CompressorId for CustomCompressor {
+///     const ID: u8 = 0;
+/// }
+///
+/// impl Compressor for CustomCompressor {
+///     fn encode(&self, block: Vec<u8>) -> rusty_leveldb::Result<Vec<u8>> {
+///         // Do something
+///         Ok(block)
+///     }
+///
+///     fn decode(&self, block: Vec<u8>) -> rusty_leveldb::Result<Vec<u8>> {
+///         // Do something
+///         Ok(block)
+///     }
+/// }
+/// ```
+///
+/// See [crate::CompressorList] for usage
 pub trait Compressor {
     fn encode(&self, block: Vec<u8>) -> crate::Result<Vec<u8>>;
 
     fn decode(&self, block: Vec<u8>) -> crate::Result<Vec<u8>>;
 }
 
+/// Set default compressor id
 pub trait CompressorId {
     const ID: u8;
 }
 
+/// A compressor that do **Nothing**
+///
+/// It default id is `0`
 #[derive(Debug, Clone, Copy, Default)]
 pub struct NoneCompressor;
 
-impl NoneCompressor {
-    pub fn new() -> Box<dyn Compressor> {
-        Box::new(Self)
-    }
-}
-
 impl CompressorId for NoneCompressor {
     const ID: u8 = 0;
 }
@@ -31,15 +55,12 @@
     }
 }
 
+/// A compressor that compress data with Google's Snappy
+///
+/// It default id is `1`
 #[derive(Debug, Clone, Copy, Default)]
 pub struct SnappyCompressor;
 
-impl SnappyCompressor {
-    pub fn new() -> Box<dyn Compressor> {
-        Box::new(Self)
-    }
-}
-
 impl CompressorId for SnappyCompressor {
     const ID: u8 = 1;
 }
--- a/src/db_impl.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/db_impl.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -110,9 +110,6 @@
     /// depends on the options set (`create_if_missing`, `error_if_exists`).
     pub fn open<P: AsRef<Path>>(name: P, opt: Options) -> Result<DB> {
         let name = name.as_ref();
-        if opt.compressor_list[opt.compressor as usize].is_none() {
-            err(StatusCode::InvalidOption, "need set compressor")?;
-        }
         let mut db = DB::new(name, opt);
         let mut ve = VersionEdit::new();
         let save_manifest = db.recover(&mut ve)?;
--- a/src/error.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/error.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -20,7 +20,6 @@
     IOError,
     InvalidArgument,
     InvalidData,
-    InvalidOption,
     LockError,
     NotFound,
     NotSupported,
--- a/src/lib.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/lib.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -87,7 +87,7 @@
 pub use error::{Result, Status, StatusCode};
 pub use filter::{BloomPolicy, FilterPolicy};
 pub use mem_env::MemEnv;
-pub use options::{in_memory, Options};
+pub use options::{in_memory, CompressorList, Options};
 pub use skipmap::SkipMap;
 pub use types::LdbIterator;
 pub use write_batch::WriteBatch;
--- a/src/options.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/options.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -1,13 +1,13 @@
 use crate::block::Block;
 use crate::cache::Cache;
 use crate::cmp::{Cmp, DefaultCmp};
-use crate::compressor::Compressor;
-use crate::disk_env;
+use crate::compressor::{self, Compressor, CompressorId};
 use crate::env::Env;
-use crate::filter;
 use crate::infolog::{self, Logger};
 use crate::mem_env::MemEnv;
 use crate::types::{share, Shared};
+use crate::{disk_env, Result};
+use crate::{filter, Status, StatusCode};
 
 use std::default::Default;
 use std::rc::Rc;
@@ -36,11 +36,13 @@
     pub block_cache: Shared<Cache<Block>>,
     pub block_size: usize,
     pub block_restart_interval: usize,
+    /// Compressor id in compressor list
+    ///
     /// Note: you have to open a database with the same compression type as it was written to, in
     /// order to not lose data! (this is a bug and will be fixed)
     pub compressor: u8,
 
-    pub compressor_list: Rc<[Option<Box<dyn Compressor>>; 256]>,
+    pub compressor_list: Rc<CompressorList>,
     pub reuse_logs: bool,
     pub reuse_manifest: bool,
     pub filter_policy: filter::BoxedFilterPolicy,
@@ -48,11 +50,6 @@
 
 impl Default for Options {
     fn default() -> Options {
-        const INIT: Option<Box<dyn Compressor>> = None;
-        let mut compressor_list = [INIT; 256];
-        compressor_list[0] = Some(crate::compressor::NoneCompressor::new());
-        compressor_list[1] = Some(crate::compressor::SnappyCompressor::new());
-
         Options {
             cmp: Rc::new(Box::new(DefaultCmp)),
             env: Rc::new(Box::new(disk_env::PosixDiskEnv::new())),
@@ -70,12 +67,64 @@
             reuse_logs: true,
             reuse_manifest: true,
             compressor: 0,
-            compressor_list: Rc::new(compressor_list),
+            compressor_list: Rc::new(CompressorList::default()),
             filter_policy: Rc::new(Box::new(filter::BloomPolicy::new(DEFAULT_BITS_PER_KEY))),
         }
     }
 }
 
+/// Customize compressor method for leveldb
+///
+/// `Default` value is like the code below
+/// ```
+/// # use rusty_leveldb::{compressor, CompressorList};
+/// let mut list = CompressorList::new();
+/// list.set(compressor::NoneCompressor);
+/// list.set(compressor::SnappyCompressor);
+/// ```
+pub struct CompressorList([Option<Box<dyn Compressor>>; 256]);
+
+impl CompressorList {
+    /// Create a **Empty** compressor list
+    pub fn new() -> Self {
+        const INIT: Option<Box<dyn Compressor>> = None;
+        Self([INIT; 256])
+    }
+
+    /// Set compressor with the id in `CompressorId` trait
+    pub fn set<T>(&mut self, compressor: T)
+    where
+        T: Compressor + CompressorId + 'static,
+    {
+        self.set_with_id(T::ID, compressor)
+    }
+
+    /// Set compressor with id
+    pub fn set_with_id(&mut self, id: u8, compressor: impl Compressor + 'static) {
+        self.0[id as usize] = Some(Box::new(compressor));
+    }
+
+    pub fn is_set(&self, id: u8) -> bool {
+        self.0[id as usize].is_some()
+    }
+
+    pub fn get(&self, id: u8) -> Result<&Box<dyn Compressor + 'static>> {
+        self.0[id as usize].as_ref().ok_or_else(|| Status {
+            code: StatusCode::InvalidData,
+            err: format!("invalid compression id `{}`", id),
+        })
+    }
+}
+
+impl Default for CompressorList {
+    fn default() -> Self {
+        let mut list = Self::new();
+        list.set(compressor::NoneCompressor);
+        list.set(compressor::SnappyCompressor);
+        list
+    }
+}
+
 /// Returns Options that will cause a database to exist purely in-memory instead of being stored on
 /// disk. This is useful for testing or ephemeral databases.
 pub fn in_memory() -> Options {
--- a/src/table_block.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/table_block.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -70,11 +70,11 @@
         );
     }
     let compressor_list = opt.compressor_list.clone();
-    if let Some(compressor) = compressor_list[compress[0] as usize].as_ref() {
-        Ok(Block::new(opt, compressor.decode(buf)?))
-    } else {
-        err(StatusCode::InvalidData, "invalid compression type")
-    }
+
+    Ok(Block::new(
+        opt,
+        compressor_list.get(compress[0])?.decode(buf)?,
+    ))
 }
 
 /// Verify checksum of block
--- a/src/table_builder.rs	Tue Jul 04 21:50:32 2023 +0800
+++ b/src/table_builder.rs	Sat Jul 15 14:05:55 2023 +0800
@@ -180,9 +180,7 @@
         let contents = block.finish();
 
         let compressor_list = self.opt.compressor_list.clone();
-        let compressor = compressor_list[self.opt.compressor as usize]
-            .as_ref()
-            .unwrap();
+        let compressor = compressor_list.get(self.opt.compressor)?;
 
         let handle = self.write_block(contents, (self.opt.compressor, compressor))?;
 
@@ -232,9 +230,7 @@
         let compressor_list = self.opt.compressor_list.clone();
         let compressor_id_pair = (
             self.opt.compressor,
-            compressor_list[self.opt.compressor as usize]
-                .as_ref()
-                .unwrap(),
+            compressor_list.get(self.opt.compressor)?,
         );
 
         // If there's a pending data block, write it
@@ -259,7 +255,7 @@
                 fblock_data,
                 (
                     compressor::NoneCompressor::ID,
-                    &compressor::NoneCompressor::new(),
+                    &(Box::new(compressor::NoneCompressor) as Box<dyn Compressor>),
                 ),
             )?;