changeset 12:1c464fb19d9f

Make Storage interface take iterator
author Lewin Bormann <lbo@spheniscida.de>
date Mon, 23 Mar 2020 14:21:47 +0100
parents b7600da13c32
children bbe33cf09e69
files src/driver.rs src/implem.rs
diffstat 2 files changed, 7 insertions(+), 7 deletions(-) [+]
line wrap: on
line diff
--- a/src/driver.rs	Mon Mar 23 14:11:43 2020 +0100
+++ b/src/driver.rs	Mon Mar 23 14:21:47 2020 +0100
@@ -13,8 +13,8 @@
 
 /// Store fetched results, which come as key/value pairs, somewhere.
 #[async_trait::async_trait]
-pub trait Storage<T> {
-    async fn store(&mut self, d: Vec<T>) ->Result<(), err::HTTPError>;
+pub trait Storage<T: Send> {
+    async fn store(&mut self, d: Box<dyn Iterator<Item=T> + Send>) ->Result<(), err::HTTPError>;
 }
 
 /// Return Uris to explore, both as initial set and for every fetched page.
@@ -49,7 +49,7 @@
     queue: Vec<Uri>,
 }
 
-impl<T> Driver<T> {
+impl<T: 'static + Send> Driver<T> {
     /// Create a new Driver instance.
     pub fn new(logic: DriverLogic<T>, https: Option<http::HTTPS>) -> Driver<T> {
         Driver { https: https.unwrap_or(http::HTTPS::new()), logic: logic, queue: Vec::with_capacity(64) }
@@ -67,7 +67,7 @@
             let resp = self.https.get(&uri).await?;
             let doc = extract::parse_response(resp)?;
             let extracted = self.logic.extract.extract(&uri, &doc);
-            self.logic.store.store(extracted);
+            self.logic.store.store(Box::new(extracted.into_iter()));
             let next = self.logic.explore.next(&uri, &doc);
             info!("Appended URIs after fetch: {:?}", next);
             self.queue.extend(next);
--- a/src/implem.rs	Mon Mar 23 14:11:43 2020 +0100
+++ b/src/implem.rs	Mon Mar 23 14:21:47 2020 +0100
@@ -25,7 +25,7 @@
         let prices = data.pop().unwrap();
         let descs = data.pop().unwrap();
 
-        let onlytext = rex::compile("^[a-zA-Z0-9\\.,+/ -]+").unwrap();
+        let onlytext = rex::compile("^([a-zA-Z0-9\\.,+/ -]+) +").unwrap();
 
         let zipped: Vec<(String, String)> = descs
             .into_iter()
@@ -86,8 +86,8 @@
 
 #[async_trait::async_trait]
 impl driver::Storage<(String,String)> for DebuggingStorage {
-    async fn store(&mut self, all: Vec<(String,String)>) -> Result<(), HTTPError> {
-        info!("STORAGE: Received {:?}", all);
+    async fn store(&mut self, all: Box<dyn Iterator<Item=(String,String)> + Send>) -> Result<(), HTTPError> {
+        info!("STORAGE: Received {:?}", all.collect::<Vec<(String,String)>>());
         Ok(())
     }
 }