Mercurial > lbo > hg > scrapeprice
changeset 12:1c464fb19d9f
Make Storage interface take iterator
author | Lewin Bormann <lbo@spheniscida.de> |
---|---|
date | Mon, 23 Mar 2020 14:21:47 +0100 |
parents | b7600da13c32 |
children | bbe33cf09e69 |
files | src/driver.rs src/implem.rs |
diffstat | 2 files changed, 7 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/src/driver.rs Mon Mar 23 14:11:43 2020 +0100 +++ b/src/driver.rs Mon Mar 23 14:21:47 2020 +0100 @@ -13,8 +13,8 @@ /// Store fetched results, which come as key/value pairs, somewhere. #[async_trait::async_trait] -pub trait Storage<T> { - async fn store(&mut self, d: Vec<T>) ->Result<(), err::HTTPError>; +pub trait Storage<T: Send> { + async fn store(&mut self, d: Box<dyn Iterator<Item=T> + Send>) ->Result<(), err::HTTPError>; } /// Return Uris to explore, both as initial set and for every fetched page. @@ -49,7 +49,7 @@ queue: Vec<Uri>, } -impl<T> Driver<T> { +impl<T: 'static + Send> Driver<T> { /// Create a new Driver instance. pub fn new(logic: DriverLogic<T>, https: Option<http::HTTPS>) -> Driver<T> { Driver { https: https.unwrap_or(http::HTTPS::new()), logic: logic, queue: Vec::with_capacity(64) } @@ -67,7 +67,7 @@ let resp = self.https.get(&uri).await?; let doc = extract::parse_response(resp)?; let extracted = self.logic.extract.extract(&uri, &doc); - self.logic.store.store(extracted); + self.logic.store.store(Box::new(extracted.into_iter())); let next = self.logic.explore.next(&uri, &doc); info!("Appended URIs after fetch: {:?}", next); self.queue.extend(next);
--- a/src/implem.rs Mon Mar 23 14:11:43 2020 +0100 +++ b/src/implem.rs Mon Mar 23 14:21:47 2020 +0100 @@ -25,7 +25,7 @@ let prices = data.pop().unwrap(); let descs = data.pop().unwrap(); - let onlytext = rex::compile("^[a-zA-Z0-9\\.,+/ -]+").unwrap(); + let onlytext = rex::compile("^([a-zA-Z0-9\\.,+/ -]+) +").unwrap(); let zipped: Vec<(String, String)> = descs .into_iter() @@ -86,8 +86,8 @@ #[async_trait::async_trait] impl driver::Storage<(String,String)> for DebuggingStorage { - async fn store(&mut self, all: Vec<(String,String)>) -> Result<(), HTTPError> { - info!("STORAGE: Received {:?}", all); + async fn store(&mut self, all: Box<dyn Iterator<Item=(String,String)> + Send>) -> Result<(), HTTPError> { + info!("STORAGE: Received {:?}", all.collect::<Vec<(String,String)>>()); Ok(()) } }