Mercurial > lbo > hg > localmr
changeset 17:5a7f556c223d
Extend documentation.
author | Lewin Bormann <lbo@spheniscida.de> |
---|---|
date | Sun, 31 Jan 2016 13:01:22 +0000 |
parents | 2b2676d774b1 |
children | 3b56d88a7ab8 |
files | src/closure_mr.rs src/formats/util.rs src/formats/writelog.rs src/mapreducer.rs |
diffstat | 4 files changed, 24 insertions(+), 2 deletions(-) [+] |
line wrap: on
line diff
--- a/src/closure_mr.rs Sun Jan 31 12:52:22 2016 +0000 +++ b/src/closure_mr.rs Sun Jan 31 13:01:22 2016 +0000 @@ -1,6 +1,9 @@ +//! A MapReducer that uses supplied map()/reduce() functions. + use mapreducer::{MEmitter, MapReducer, MapperF, MultiRecord, REmitter, Record, ReducerF}; -struct ClosureMapReducer { +/// Use your functions in a MapReduce (instead of implementing your own mapreducer) +pub struct ClosureMapReducer { mapper: MapperF, reducer: ReducerF, } @@ -15,6 +18,7 @@ } impl ClosureMapReducer { + /// Create a new MapReducer from the supplied functions. pub fn new(mapper: MapperF, reducer: ReducerF) -> ClosureMapReducer { ClosureMapReducer { mapper: mapper,
--- a/src/formats/util.rs Sun Jan 31 12:52:22 2016 +0000 +++ b/src/formats/util.rs Sun Jan 31 13:01:22 2016 +0000 @@ -1,3 +1,5 @@ +//! Various iterators/adapters used for input/output formats. + use mapreducer::Record; use std::fmt;
--- a/src/formats/writelog.rs Sun Jan 31 12:52:22 2016 +0000 +++ b/src/formats/writelog.rs Sun Jan 31 13:01:22 2016 +0000 @@ -1,3 +1,6 @@ +//! WriteLogs are sequences of length-prefixed records of arbitrary data +//! in a file. + #![allow(dead_code)] use std::io::{Result, Write, Read};
--- a/src/mapreducer.rs Sun Jan 31 12:52:22 2016 +0000 +++ b/src/mapreducer.rs Sun Jan 31 13:01:22 2016 +0000 @@ -1,12 +1,16 @@ +//! The MapReducer trait and associated types. + use std::collections::LinkedList; use std::clone::Clone; +/// A (key,value) pair. pub struct Record { pub key: String, pub value: String, } -/// Input to a reducer function. +/// A (key,[value]) pair; typicall used as input to a reducer function. +/// Can be easily iterated over, e.g. in a `for` loop. pub struct MultiRecord { key: String, value: Box<Iterator<Item = String>>, @@ -28,6 +32,7 @@ } } +/// Emitter type used in the mapper phase; used to emit (key,value) pairs. pub struct MEmitter { r: LinkedList<Record>, } @@ -47,6 +52,7 @@ } } +/// Emitter used in the reducer phase; used to emit values. pub struct REmitter { r: LinkedList<String>, } @@ -63,13 +69,20 @@ } } +/// Map() function type. The MEmitter argument is used to emit values from +/// the map() function. pub type MapperF = fn(&mut MEmitter, Record); +/// Reduce() function type. The REmitter argument is used to emit values +/// from the reduce() function. pub type ReducerF = fn(&mut REmitter, MultiRecord); /// A type implementing map() and reduce() functions. +/// The MapReducer is cloned once per mapper/reducer thread. pub trait MapReducer: Clone { /// Takes one <key,value> pair and an emitter. /// The emitter is used to yield results from the map phase. fn map(&self, em: &mut MEmitter, record: Record); + /// Takes one key and one or more values and emits one or more + /// values. fn reduce(&self, em: &mut REmitter, records: MultiRecord); }