changeset 25:68f181727187

Add shard() function and default impl to MapReducer
author Lewin Bormann <lbo@spheniscida.de>
date Sun, 31 Jan 2016 16:19:51 +0000
parents 64d2dfde95a5
children 5a61cde90fe6
files src/lib.rs src/mapreducer.rs
diffstat 2 files changed, 12 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib.rs	Sun Jan 31 15:00:03 2016 +0000
+++ b/src/lib.rs	Sun Jan 31 16:19:51 2016 +0000
@@ -4,6 +4,7 @@
 
 pub mod closure_mr;
 pub mod formats;
+pub mod map;
 pub mod mapreducer;
 
 
--- a/src/mapreducer.rs	Sun Jan 31 15:00:03 2016 +0000
+++ b/src/mapreducer.rs	Sun Jan 31 16:19:51 2016 +0000
@@ -2,6 +2,7 @@
 
 use std::collections::LinkedList;
 use std::clone::Clone;
+use std::hash::{Hasher, SipHasher};
 
 /// A (key,value) pair.
 pub struct Record {
@@ -85,4 +86,14 @@
     /// Takes one key and one or more values and emits one or more
     /// values.
     fn reduce(&self, em: &mut REmitter, records: MultiRecord);
+
+    /// Determines how to map keys to (reduce) shards.
+    /// Returns a number in [0; n) determining the shard the key belongs in.
+    /// The default implementation uses a simple hash (SipHasher) and modulo.
+    fn shard(n: u32, key: &String) -> u32 {
+        let mut h = SipHasher::new();
+        h.write(key.as_bytes());
+        h.finish() as u32 % n
+    }
 }
+