Cookie stuff - JWT signer, anonymous visitor cookies, etc.

- JWT signer is reusable, tested

- AVT cookies improve the anonymous user experience and will help
  internal analytics

Cargo.lock

@@ -1420,6 +1420,16 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "cookies"
+version = "0.0.1"
+dependencies = [
+ "errors",
+ "hmac 0.10.1",
+ "jwt",
+ "sha2 0.9.5",
+]
+
 [[package]]
 name = "copyless"
 version = "0.1.5"
@@ -5551,6 +5561,7 @@ dependencies = [
  "cloud_storage",
  "config",
  "container_common",
+ "cookies",
  "data-encoding",
  "datetimes",
  "derive_more",

Cargo.toml

@@ -24,6 +24,7 @@ members = [
   "crates/lib/caching/memory_caching",
   "crates/lib/caching/redis_caching",
   "crates/lib/concurrency",
+  "crates/lib/cookies",
   "crates/lib/crockford",
   "crates/lib/datetimes",
   "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose

crates/_types/tokens/src/prefixes.rs

@@ -13,6 +13,8 @@ use strum::EnumIter;
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 pub(crate) enum EntityType {
+  /// AVTs are not stored as primary keys in any table, but an index in many tables.
+  AnonymousVisitorTracking,
   AuditLog,
   Comment,
   DownloadJob,
@@ -36,6 +38,7 @@ impl EntityType {
 
   pub fn prefix(self) -> &'static str {
     match self {
+      Self::AnonymousVisitorTracking => "avt_",
       Self::AuditLog => "audit_",
       Self::Comment => "comment_",
       Self::DownloadJob => "jdown_", // NB: Was "JGUP:"

crates/_types/tokens/src/tokens/anonymous_visitor_tracking.rs

@@ -0,0 +1,16 @@
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::prefixes::EntityType;
+
+/// A rare instance that is *not* a primary key.
+/// These are generated as cookies to track users for a better logged out experience and for analytics.
+/// We use these as indices into several columns (ML result types, uploads, etc.)
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
+#[sqlx(transparent)]
+pub struct AnonymousVisitorTrackingToken(pub String);
+
+impl_string_token!(AnonymousVisitorTrackingToken);
+impl_crockford_generator!(AnonymousVisitorTrackingToken, 32usize, EntityType::AnonymousVisitorTracking, CrockfordLower);

crates/_types/tokens/src/tokens/mod.rs

@@ -1,3 +1,4 @@
+pub mod anonymous_visitor_tracking;
 pub mod audit_logs;
 pub mod comments;
 pub mod generic_download_jobs;

crates/_types/tokens/src/tokens/users.rs

@@ -5,11 +5,6 @@ use serde::Serialize;
 
 use crate::prefixes::EntityType;
 
-// FIXME: I think that this file+module structure is an anti-pattern.
-//  In the future, we should create two top-level directories: /tokens and /ids, and each token or
-//  ID type should be in its own file. (NB: single files cannot share token definitions due to the
-//  macro generating test modules that would conflict.)
-
 /// The primary key for users.
 #[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
 #[sqlx(transparent)]

crates/lib/cookies/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "cookies"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+# NB: Public "cookies" crate is version 0.0.1 from 10 years ago!
+name = "cookies"
+path = "src/lib.rs"
+
+[dependencies]
+# Internal
+errors = { path = "../errors" }
+
+# External
+hmac = "0.10.0" # jwt signing
+jwt = "0.13.0"
+sha2 = "0.9.5" # hmac signing
+
+[dev-dependencies]
+# None yet

crates/lib/cookies/src/jwt_signer.rs

@@ -0,0 +1,95 @@
+use std::collections::BTreeMap;
+
+use hmac::Hmac;
+use hmac::NewMac;
+use jwt::SignWithKey;
+use jwt::VerifyWithKey;
+use sha2::Sha256;
+
+use errors::{anyhow, AnyhowResult};
+
+#[derive(Clone)]
+pub struct JwtSigner {
+  hmac_key: Hmac<Sha256>,
+}
+
+impl JwtSigner {
+
+  pub fn new(hmac_secret: &str) -> AnyhowResult<Self> {
+    let hmac_key = Hmac::new_varkey(hmac_secret.as_bytes())
+        .map_err(|e| anyhow!("invalid hmac: {:?}", e))?;
+
+    Ok(Self {
+      hmac_key,
+    })
+  }
+
+  /// Turn a map of claims into a signed JWT cookie payload (string)
+  pub fn claims_to_jwt(&self, claims: &impl Claimable) -> AnyhowResult<String> {
+    let jwt_string = claims.sign(&self.hmac_key)?;
+    Ok(jwt_string)
+  }
+
+  /// Turn a JWT cookie payload (string) into a map of (key, value) claims
+  pub fn jwt_to_claims(&self, jwt: &str) -> AnyhowResult<BTreeMap<String, String>> {
+    let claims = jwt.verify_with_key(&self.hmac_key)?;
+    Ok(claims)
+  }
+}
+
+pub trait Claimable {
+  fn sign(&self, hmac_key: &Hmac<Sha256>) -> AnyhowResult<String>;
+}
+
+impl Claimable for BTreeMap<String, String> {
+  fn sign(&self, hmac_key: &Hmac<Sha256>) -> AnyhowResult<String> {
+    let claims = self.sign_with_key(hmac_key)?;
+    Ok(claims)
+  }
+}
+
+impl Claimable for BTreeMap<&str, &str> {
+  fn sign(&self, hmac_key: &Hmac<Sha256>) -> AnyhowResult<String> {
+    let claims = self.sign_with_key(hmac_key)?;
+    Ok(claims)
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::collections::BTreeMap;
+
+  use crate::jwt_signer::JwtSigner;
+
+  #[test]
+  fn test_round_trip() {
+    let signer = JwtSigner::new("secret").unwrap();
+
+    let mut claims = BTreeMap::new();
+    claims.insert("foo", "yes");
+    claims.insert("bar", "no");
+
+    let jwt = signer.claims_to_jwt(&claims).unwrap();
+
+    let output_claims = signer.jwt_to_claims(&jwt).unwrap();
+
+    assert_eq!(claims.len(), output_claims.len());
+
+    for (k, v) in output_claims.iter() {
+      assert_eq!(v, claims[k.as_str()]);
+    }
+  }
+
+  #[test]
+  fn test_stability() {
+    let signer = JwtSigner::new("secret").unwrap();
+
+    let mut claims = BTreeMap::new();
+    claims.insert("foo", "yes");
+    claims.insert("bar", "no");
+
+    let jwt = signer.claims_to_jwt(&claims).unwrap();
+
+    assert_eq!("eyJhbGciOiJIUzI1NiJ9.eyJiYXIiOiJubyIsImZvbyI6InllcyJ9.k4eRvIrtw4tuGDtQqSmEDngtGYAaQVRYVMCK5N3JcZA", &jwt);
+  }
+}

crates/lib/cookies/src/lib.rs

@@ -0,0 +1,20 @@
+//! cookies
+//!
+//! This library contains reusable, tested, cross-cutting cookie functionality.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod jwt_signer;

crates/service/plugins/users/src/utils/session_cookie_manager.rs

@@ -70,12 +70,12 @@ impl SessionCookieManager {
       && !self.cookie_domain.to_lowercase().contains("localhost");
 
     Ok(Cookie::build(SESSION_COOKIE_NAME, jwt_string)
-      //.domain(&self.cookie_domain)
       .secure(make_secure) // HTTPS-only
+      .permanent()
+      //.domain(&self.cookie_domain)
       //.path("/")
       //.http_only(true) // Not exposed to Javascript
       //.expires(OffsetDateTime::now_utc() + time::Duration::days(365))
-      .permanent()
       //.same_site(SameSite::Lax)
       .finish())
   }

crates/service/web/storyteller_web/Cargo.toml

@@ -22,6 +22,7 @@ buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
 config = { path = "../../../lib/deprecated/config" }
 container_common = { path = "../../../lib/deprecated/container_common" }
+cookies = { path = "../../../lib/cookies" }
 datetimes = { path = "../../../lib/datetimes" }
 easyenv = { path = "../../../lib/easyenv" }
 enums = { path = "../../../_types/enums" }

crates/service/web/storyteller_web/src/cookies/anonymous_visitor_tracking/avt_cookie_manager.rs

@@ -0,0 +1,78 @@
+use actix_web::cookie::Cookie;
+use actix_web::HttpRequest;
+use anyhow::anyhow;
+use log::warn;
+
+use cookies::jwt_signer::JwtSigner;
+use errors::AnyhowResult;
+
+use crate::cookies::anonymous_visitor_tracking::avt_cookie_payload::AvtCookiePayload;
+
+const VISITOR_COOKIE_NAME : &str = "visitor";
+
+/// Handle "anonymous visitor tracking" cookies.
+/// This enables us to associate results with an anonymous user for a better experience,
+/// as well as do some form of return visitor tracking.
+pub struct AvtCookieManager {
+  cookie_domain: String,
+  jwt_signer: JwtSigner,
+}
+
+impl AvtCookieManager {
+
+  pub fn new(cookie_domain: &str, hmac_secret: &str) -> AnyhowResult<Self> {
+    Ok(Self {
+      cookie_domain: cookie_domain.to_string(),
+      jwt_signer: JwtSigner::new(hmac_secret)?,
+    })
+  }
+
+  pub fn make_new_cookie(&self) -> AnyhowResult<Cookie> {
+    let payload = AvtCookiePayload::new();
+    let claims = payload.to_map();
+    let jwt_string = self.jwt_signer.claims_to_jwt(&claims)?;
+
+    let make_secure = !self.cookie_domain.to_lowercase().contains("jungle.horse")
+        && !self.cookie_domain.to_lowercase().contains("localhost");
+
+    Ok(Cookie::build(VISITOR_COOKIE_NAME, jwt_string)
+        .secure(make_secure) // HTTPS-only
+        .permanent()
+        //.domain(&self.cookie_domain)
+        //.path("/")
+        //.http_only(true) // Not exposed to Javascript
+        //.expires(OffsetDateTime::now_utc() + time::Duration::days(365))
+        //.same_site(SameSite::Lax)
+        .finish())
+  }
+
+  pub fn make_delete_cookie(&self) -> Cookie {
+    let mut cookie = Cookie::build(VISITOR_COOKIE_NAME, "DELETED")
+        .expires(actix_web::cookie::time::OffsetDateTime::UNIX_EPOCH)
+        .finish();
+    cookie.make_removal();
+    cookie
+  }
+
+  pub fn decode_cookie_payload(&self, visitor_cookie: &Cookie) -> AnyhowResult<AvtCookiePayload> {
+    let cookie_contents = visitor_cookie.value().to_string();
+    let claims = self.jwt_signer.jwt_to_claims(&cookie_contents)?;
+    let payload = AvtCookiePayload::from_map(claims)?;
+    Ok(payload)
+  }
+
+  pub fn decode_cookie_payload_from_request(&self, request: &HttpRequest) -> AnyhowResult<Option<AvtCookiePayload>> {
+    let cookie = match request.cookie(VISITOR_COOKIE_NAME) {
+      None => return Ok(None),
+      Some(cookie) => cookie,
+    };
+
+    match self.decode_cookie_payload(&cookie) {
+      Err(e) => {
+        warn!("Visitor cookie decode error: {:?}", e);
+        Err(anyhow!("Could not decode visitor cookie: {:?}", e))
+      },
+      Ok(payload) => Ok(Some(payload)),
+    }
+  }
+}

crates/service/web/storyteller_web/src/cookies/anonymous_visitor_tracking/avt_cookie_payload.rs

@@ -0,0 +1,94 @@
+use std::collections::BTreeMap;
+use std::str::FromStr;
+
+use anyhow::anyhow;
+
+use errors::AnyhowResult;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+
+/**
+ * Cookie version history
+ *
+ *  Version 1: Claims include "avt_token" and "cookie_version"
+ */
+const COOKIE_VERSION : u32 = 1;
+
+// TODO: Should probably use protobuf or nom for handling
+//  data-migration sensitive wire formats rather than
+//  hand-rolling these.
+
+/// The payload of AVT cookies.
+///
+///  Version 1 - only `avt_token` and `version`.
+#[derive(Clone, Eq, PartialEq, Debug)]
+pub struct AvtCookiePayload {
+  // TODO: Strongly type
+  pub avt_token: AnonymousVisitorTrackingToken,
+  pub cookie_version: u32,
+}
+
+impl AvtCookiePayload {
+
+  pub fn new() -> Self {
+    let avt_token = AnonymousVisitorTrackingToken::generate();
+
+    AvtCookiePayload {
+      avt_token,
+      cookie_version: COOKIE_VERSION,
+    }
+  }
+
+  pub fn from_map(map: BTreeMap<String, String>) -> AnyhowResult<Self> {
+    let avt_token = map.get("avt_token")
+        .ok_or(anyhow!("avt_token not present"))?
+        .to_string();
+
+    let cookie_version = map
+        .get("cookie_version")
+        .ok_or(anyhow!("avt_token not present"))?;
+
+    let avt_token = AnonymousVisitorTrackingToken::new_from_str(&avt_token);
+
+    let cookie_version = u32::from_str(cookie_version)?;
+
+    Ok(AvtCookiePayload {
+      avt_token,
+      cookie_version,
+    })
+  }
+
+  pub fn to_map(&self) -> BTreeMap<String, String> {
+    let mut map = BTreeMap::new();
+    map.insert("avt_token".to_string(), self.avt_token.to_string());
+    map.insert("cookie_version".to_string(), self.cookie_version.to_string());
+    map
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+  use crate::cookies::anonymous_visitor_tracking::avt_cookie_payload::AvtCookiePayload;
+
+  #[test]
+  fn test_new() {
+    let payload = AvtCookiePayload::new();
+
+    assert_eq!(payload.cookie_version, 1);
+    assert_eq!(payload.avt_token.0.len(), 32);
+    assert!(payload.avt_token.to_string().starts_with("avt_"));
+  }
+
+  #[test]
+  fn round_trip_test() {
+    let payload = AvtCookiePayload {
+      avt_token: AnonymousVisitorTrackingToken::generate(),
+      cookie_version: 123,
+    };
+
+    let round_trip_payload= AvtCookiePayload::from_map(payload.to_map())
+        .unwrap();
+
+    assert_eq!(&payload, &round_trip_payload);
+  }
+}

crates/service/web/storyteller_web/src/cookies/anonymous_visitor_tracking/mod.rs

@@ -0,0 +1,2 @@
+pub mod avt_cookie_manager;
+pub mod avt_cookie_payload;

crates/service/web/storyteller_web/src/cookies/mod.rs

@@ -0,0 +1 @@
+pub mod anonymous_visitor_tracking;

crates/service/web/storyteller_web/src/main.rs

@@ -90,6 +90,7 @@ pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/bin
 
 pub mod billing;
 pub mod configs;
+pub mod cookies;
 pub mod http_server;
 pub mod memory_cache;
 pub mod model;