changeset 50:603c38d64cd5

Experiment: Pre-fetch larger buffers from shards
author Lewin Bormann <lbo@spheniscida.de>
date Wed, 03 Feb 2016 21:15:39 +0000
parents 8a2538cbac8a
children d601cf5388d3
files src/shard_merge.rs
diffstat 1 files changed, 25 insertions(+), 24 deletions(-) [+]
line wrap: on
line diff
--- a/src/shard_merge.rs	Wed Feb 03 20:50:34 2016 +0000
+++ b/src/shard_merge.rs	Wed Feb 03 21:15:39 2016 +0000
@@ -7,6 +7,7 @@
 
 use std::cmp::{Ord, Ordering};
 use std::iter;
+use std::collections::LinkedList;
 
 /// Function type to be used as custom compare function
 /// (rust's standard String comparison is based on ASCII values, not dictionary order)
@@ -19,43 +20,43 @@
     left: Box<Iterator<Item = T> + 'a>,
     right: Box<Iterator<Item = T> + 'a>,
 
-    left_peeked: Option<T>,
-    right_peeked: Option<T>,
+    left_buf: LinkedList<T>,
+    right_buf: LinkedList<T>,
     comparer: Option<Comparer<T>>,
 }
 
+fn fill_n<T, I: Iterator<Item=T>>(list: &mut LinkedList<T>, from: &mut I, n: i32) {
+    for _ in 0..n {
+        match from.next() {
+            None => return,
+            Some(t) => list.push_back(t)
+        }
+    }
+}
+
 impl<'a, T: Ord + Clone> Iterator for ShardMergeIterator<'a, T> {
     type Item = T;
     fn next(&mut self) -> Option<Self::Item> {
         // fill up
-        match (self.left_peeked.clone(), self.right_peeked.clone()) {
-            (None, None) => {
-                self.left_peeked = self.left.next();
-                self.right_peeked = self.right.next()
-            }
-            (Some(_), None) => self.right_peeked = self.right.next(),
-            (None, Some(_)) => self.left_peeked = self.left.next(),
-            (Some(_), Some(_)) => (),
+        match (self.left_buf.len(), self.right_buf.len()) {
+            (0, 0) => { fill_n(&mut self.left_buf, &mut self.left, 16); fill_n(&mut self.right_buf, &mut self.right, 16); },
+            (0, _) => { fill_n(&mut self.left_buf, &mut self.left, 16); },
+            (_, 0) => { fill_n(&mut self.right_buf, &mut self.right, 16); },
+            (_, _) => (),
         }
 
-        // Consume peeked values
-        match (self.left_peeked.clone(), self.right_peeked.clone()) {
+        let (left, right) = (self.left_buf.pop_front(), self.right_buf.pop_front());
+        match (left, right) {
             (None, None) => return None,
-            (l @ Some(_), None) => {
-                self.left_peeked = None;
-                return l;
-            }
-            (None, r @ Some(_)) => {
-                self.right_peeked = None;
-                return r;
-            }
+            (l @ Some(_), None) => return l,
+            (None, r @ Some(_)) => return r,
             (Some(l), Some(r)) => {
                 let cmp = self.compare(&l, &r);
                 if cmp == Ordering::Less || cmp == Ordering::Equal {
-                    self.left_peeked = None;
+                    self.right_buf.push_front(r);
                     return Some(l);
                 } else {
-                    self.right_peeked = None;
+                    self.left_buf.push_front(l);
                     return Some(r);
                 }
             }
@@ -70,8 +71,8 @@
         ShardMergeIterator {
             left: Box::new(iter::empty()),
             right: Box::new(iter::empty()),
-            left_peeked: None,
-            right_peeked: None,
+            left_buf: LinkedList::new(),
+            right_buf: LinkedList::new(),
             comparer: None,
         }
     }