changeset 52:ce8e102ee2e0

Display top referers
author Lewin Bormann <lbo@spheniscida.de>
date Thu, 21 Jul 2022 19:09:52 -0700
parents da66084c708e
children f81d2fdcbe63
files assets/index.html.hbs src/logsdb.rs src/main.rs
diffstat 3 files changed, 110 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/assets/index.html.hbs	Thu Jul 21 10:13:50 2022 -0700
+++ b/assets/index.html.hbs	Thu Jul 21 19:09:52 2022 -0700
@@ -101,15 +101,16 @@
     <div class="plotrow row3">
         <div class="plotframe halfwidth">Sessions by country
             <div class="chartcontainer2"><canvas id="sessionsByCountry"></canvas></div> </div>
-        <div class="plotframe halfwidth">Requests per Session
-            <div class="chartcontainer2"><canvas id="BLAH"></canvas></div> </div>
+        <div class="plotframe halfwidth">Top External Referers
+            <div class="chartcontainer2"><canvas id="topRefer"></canvas></div> </div>
     </div>
 
     <script>
         let plots = {"visitsAndSessions": {{{ chartconfig.visitsAndSessions }}},
                      "topPaths": {{{ chartconfig.topPaths }}},
                      "requestsBySession": {{{ chartconfig.requestsBySession }}},
-                     "sessionsByCountry": {{{ chartconfig.sessionsByCountry }}} };
+                     "sessionsByCountry": {{{ chartconfig.sessionsByCountry }}},
+                     "topRefer": {{{ chartconfig.topRefer }}} };
 
         Object.keys(plots).forEach((cv) => {
                     if (plots[cv] == undefined) {
--- a/src/logsdb.rs	Thu Jul 21 10:13:50 2022 -0700
+++ b/src/logsdb.rs	Thu Jul 21 19:09:52 2022 -0700
@@ -5,10 +5,13 @@
 use time::{Duration, OffsetDateTime};
 
 use rocket::futures::{future::ready, StreamExt};
+use rocket::http::hyper::uri::Uri;
 use rocket_db_pools::sqlx::{Executor, Row, Sqlite, SqlitePool};
 use rocket_db_pools::{Connection, Database, Pool};
 use sqlx::prelude::FromRow;
 
+use std::collections::HashMap;
+
 #[derive(Database)]
 #[database("sqlite_logs")]
 pub struct LogsDB(pub PoolType);
@@ -292,4 +295,67 @@
             .collect()
             .await)
     }
+
+    pub async fn query_top_refer_domains(
+        &mut self,
+        ctx: &LogsQueryContext,
+    ) -> Result<Vec<(String, i64)>, Error> {
+        let tz_offset = ctx.tz_offset;
+        let include_bots = if ctx.include_bots { 1 } else { 0 };
+        let result = sqlx::query(
+            r#"
+SELECT refer, count(*)
+FROM RequestLog
+JOIN Sessions ON (RequestLog.session = Sessions.id)
+WHERE instr(RequestLog.refer, RequestLog.domain) = 0 AND
+    RequestLog.atime+? > ? AND RequestLog.atime+? < ? AND
+    Sessions.is_bot <= ? AND
+    RequestLog.domain LIKE ?
+GROUP BY refer;
+"#,
+        )
+        .bind(tz_offset)
+        .bind(ctx.from.unix_timestamp())
+        .bind(tz_offset)
+        .bind(ctx.to.unix_timestamp())
+        .bind(include_bots)
+        .bind(ctx.domain.as_ref().map(String::as_str).unwrap_or("%"))
+        .fetch(&mut *self.0);
+
+        let mut by_origin = HashMap::<String, i64>::new();
+
+        result
+            .take_while(|row| {
+                if let Err(ref e) = row {
+                    error!("query_top_refer_domains: Unable to fetch row: {}", e);
+                    ready(false)
+                } else {
+                    ready(true)
+                }
+            })
+            .filter(|row| ready(row.is_ok()))
+            .map(|row| {
+                let row = row.unwrap();
+                (row.get(0), row.get(1))
+            })
+            .map(|(refer, count): (String, i64)| {
+                if let Ok(u) = refer.parse::<Uri>() {
+                    by_origin
+                        .entry(u.host().map(str::to_string).unwrap_or(String::new()))
+                        .and_modify(|c| *c += count)
+                        .or_insert(count);
+                } else if refer.is_empty() {
+                    by_origin
+                        .entry(String::new())
+                        .and_modify(|c| *c += count)
+                        .or_insert(count);
+                }
+            })
+            .collect::<()>()
+            .await;
+
+        let mut by_origin_vec = by_origin.into_iter().collect::<Vec<(String, i64)>>();
+        by_origin_vec.sort_by_key(|(_, v)| -*v);
+        Ok(by_origin_vec)
+    }
 }
--- a/src/main.rs	Thu Jul 21 10:13:50 2022 -0700
+++ b/src/main.rs	Thu Jul 21 19:09:52 2022 -0700
@@ -126,7 +126,7 @@
 
 #[derive(Default)]
 struct ChartOptions {
-    typ: String,               // bar/line/etc.
+    typ: String,                // bar/line/etc.
     index_axis: Option<String>, // x/y
     stack: Option<String>,
 }
@@ -140,8 +140,16 @@
         "red", "blue", "orange", "green", "gray", "purple", "black", "brown",
     ];
 
-    let bordercolor = if opt.typ == "line" { colors.clone() } else { vec!["white"] };
-    let bgcolor = if opt.typ != "line" { colors.clone() } else { vec![] };
+    let bordercolor = if opt.typ == "line" {
+        colors.clone()
+    } else {
+        vec!["white"]
+    };
+    let bgcolor = if opt.typ != "line" {
+        colors.clone()
+    } else {
+        vec![]
+    };
 
     if values.len() > colors.len() {
         error!("Not enough colors for line chart!");
@@ -169,25 +177,14 @@
             "scales": { "y": { "beginAtZero": true }},
             "indexAxis": opt.index_axis,
             "stack": opt.stack.as_ref().map(String::as_str),
+            "plugins": {
+                "legend": { "display": opt.typ != "bar" || opt.index_axis.as_ref().map(String::as_str) != Some("y") },
+            },
         },
     });
     inner
 }
 
-fn construct_index_querystring(
-    domain: Option<&str>,
-    from: OffsetDateTime,
-    duration: Duration,
-) -> String {
-    format!(
-        "?domain={}&from={}&duration={}",
-        domain.unwrap_or(""),
-        from.format(&time::format_description::well_known::Iso8601::DEFAULT)
-            .unwrap_or("".to_string()),
-        duration.whole_days()
-    )
-}
-
 /// Main analytics page for logged-in users.
 #[rocket::get("/?<domain>&<from>&<duration>&<includebots>", rank = 1)]
 async fn route_index_loggedin(
@@ -321,6 +318,25 @@
             "undefined".to_string()
         }
     };
+    let toprefer = match LogsDBSession(&mut conn).query_top_refer_domains(&ctx).await {
+        Ok(rs) => create_chart(
+            rs.iter().map(|(dom, ct)| dom).collect(),
+            vec![(
+                "Top External Referers".into(),
+                rs.iter().map(|(dom, ct)| ct).collect(),
+            )],
+            &ChartOptions {
+                typ: "bar".into(),
+                index_axis: Some("y".into()),
+                ..ChartOptions::default()
+            },
+        )
+        .to_string(),
+        Err(e) => {
+            error!("Couldn't build chart: {}", e);
+            "undefined".to_string()
+        }
+    };
 
     let ymd_format = time::format_description::parse("[year]-[month]-[day]").unwrap();
     let tmpl_today = begin.date().format(&ymd_format).unwrap();
@@ -337,7 +353,13 @@
             ("today", tmpl_today.as_str()),
             ("duration", tmpl_duration.as_str())].into_iter()),
         beginnav: vec![HashMap::<&str, &str>::from_iter([("link", "/abc"), ("title", "3d")].into_iter())],
-        chartconfig: context![ visitsAndSessions: vissess, topPaths: toppaths, requestsBySession: reqbyses, sessionsByCountry: sesbycountry ]],
+        chartconfig: context![
+            visitsAndSessions: vissess,
+            topPaths: toppaths,
+            requestsBySession: reqbyses,
+            sessionsByCountry: sesbycountry,
+            topRefer: toprefer, ]
+        ],
     )
 }