Mercurial > lbo > hg > localmr
changeset 25:68f181727187
Add shard() function and default impl to MapReducer
author | Lewin Bormann <lbo@spheniscida.de> |
---|---|
date | Sun, 31 Jan 2016 16:19:51 +0000 |
parents | 64d2dfde95a5 |
children | 5a61cde90fe6 |
files | src/lib.rs src/mapreducer.rs |
diffstat | 2 files changed, 12 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib.rs Sun Jan 31 15:00:03 2016 +0000 +++ b/src/lib.rs Sun Jan 31 16:19:51 2016 +0000 @@ -4,6 +4,7 @@ pub mod closure_mr; pub mod formats; +pub mod map; pub mod mapreducer;
--- a/src/mapreducer.rs Sun Jan 31 15:00:03 2016 +0000 +++ b/src/mapreducer.rs Sun Jan 31 16:19:51 2016 +0000 @@ -2,6 +2,7 @@ use std::collections::LinkedList; use std::clone::Clone; +use std::hash::{Hasher, SipHasher}; /// A (key,value) pair. pub struct Record { @@ -85,4 +86,14 @@ /// Takes one key and one or more values and emits one or more /// values. fn reduce(&self, em: &mut REmitter, records: MultiRecord); + + /// Determines how to map keys to (reduce) shards. + /// Returns a number in [0; n) determining the shard the key belongs in. + /// The default implementation uses a simple hash (SipHasher) and modulo. + fn shard(n: u32, key: &String) -> u32 { + let mut h = SipHasher::new(); + h.write(key.as_bytes()); + h.finish() as u32 % n + } } +