consolidate cloudflare error handling

Cargo.lock

@@ -1918,6 +1918,13 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "cloudflare_errors"
+version = "0.0.1"
+dependencies = [
+ "workspace-build-acceleration",
+]
+
 [[package]]
 name = "coarsetime"
 version = "0.1.34"
@@ -6841,6 +6848,7 @@ dependencies = [
  "anyhow",
  "base64 0.22.1",
  "chrono",
+ "cloudflare_errors",
  "cookie 0.18.1",
  "errors",
  "idempotency",
@@ -9634,7 +9642,6 @@ dependencies = [
  "sqlx",
  "tempfile",
  "tokens",
- "tokio",
  "workspace-build-acceleration",
 ]
 
@@ -9992,6 +9999,7 @@ dependencies = [
  "artcraft_api_defs",
  "backoff",
  "chrono",
+ "cloudflare_errors",
  "cookie 0.18.1",
  "enums",
  "enums_public",

Cargo.toml

@@ -40,6 +40,7 @@ members = [
   "crates/lib/caching/memory_caching",
   "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
+  "crates/lib/cloudflare_errors",
   "crates/lib/concurrency",
   "crates/lib/cookies",
   "crates/lib/data/collections",
@@ -216,6 +217,7 @@ url = { version = "2.5.4", features = ["serde"] }
 artcraft_api_defs = { path = "crates/schema/public/api/artcraft_api_defs" }
 build_metadata = { path = "crates/lib/build_metadata" }
 cloud_storage = { path = "crates/api_clients/cloud_storage" }
+cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
 crockford = { path = "crates/lib/data/crockford" }
 enums = { path = "crates/schema/public/enums" }
 enums_public = { path = "crates/schema/public/enums_public" }

crates/api_clients/openai_sora_client/Cargo.toml

@@ -23,6 +23,7 @@ idempotency.workspace = true
 anyhow.workspace = true
 base64.workspace = true
 chrono.workspace = true
+cloudflare_errors.workspace = true
 cookie.workspace = true
 log.workspace = true
 once_cell.workspace = true

crates/api_clients/openai_sora_client/src/sora_error.rs

@@ -1,3 +1,4 @@
+use cloudflare_errors::cloudflare_error::CloudflareError;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 use std::io;
@@ -42,15 +43,9 @@ pub enum SoraError {
   /// Typically served from Cloudflare
   /// We preserve the message for debugging.
   BadGateway(String),
-  
-  /// HTTP 403 + Cloudflare Challenge Interstitial
-  /// Cloudflare decided to block the request and present a challenge.
-  CloudFlareChallenge,
-  
-  /// HTTP 524 (Cloudflare-specific)
-  /// Cloudflare formed a TCP connection to Sora, but no payload was delivered before timeout
-  /// We preserve the message for debugging
-  CloudFlareTimeout(String),
+
+  /// Specific Cloudflare errors.
+  CloudflareError(CloudflareError),
 
   /// Another error occurred.
   OtherBadStatus(anyhow::Error),
@@ -72,6 +67,17 @@ pub enum SoraError {
   JsonErrorWithBody(serde_json::Error, String),
 }
 
+impl SoraError {
+  pub fn is_sora_having_downtime_issues(&self) -> bool {
+    match self {
+      Self::BadGateway(_) => true,
+      Self::CloudflareError(CloudflareError::GatewayTimeout504) => true,
+      Self::CloudflareError(CloudflareError::TimeoutOccurred524) => true,
+      _ => false,
+    }
+  }
+}
+
 impl Error for SoraError {}
 
 impl Display for SoraError {
@@ -98,11 +104,8 @@ impl Display for SoraError {
       Self::BadGateway(message) => {
         write!(f, "Bad Gateway; Sora is likely having issues: {:?}", message)
       }
-      Self::CloudFlareChallenge => {
-        write!(f, "Cloudflare Challenge (403); Cloudflare wants to verify the request with a CAPTCHA challenge.")
-      }
-      Self::CloudFlareTimeout(message) => {
-        write!(f, "Cloudflare Timeout (524); Sora is likely having issues: {:?}", message)
+      Self::CloudflareError(error) => {
+        write!(f, "Cloudflare error: {}", error)
       }
       Self::OtherBadStatus(err) => {
         write!(f, "Other error: {}", err)
@@ -149,3 +152,9 @@ impl From<serde_json::Error> for SoraError {
     Self::JsonError(err)
   }
 }
+
+impl From<CloudflareError> for SoraError {
+  fn from(err: CloudflareError) -> Self {
+    Self::CloudflareError(err)
+  }
+}

crates/api_clients/openai_sora_client/src/utils/classify_general_http_status_code_and_body.rs

@@ -1,6 +1,6 @@
 use crate::sora_error::SoraError;
+use cloudflare_errors::filter_cloudflare_errors::filter_cloudflare_errors;
 use reqwest::StatusCode;
-
 /*
 
 Old cookie - 
@@ -50,32 +50,15 @@ pub async fn classify_general_http_status_code_and_body(status: StatusCode, resp
   }
 
   let status_code = status.as_u16();
-  
+
+  if let Err(err) = filter_cloudflare_errors(status_code, &response_body) {
+    return SoraError::CloudflareError(err);
+  }
+
   match status_code {
-    403 => {
-      if message.contains("challenge-platform") 
-          || message.contains("challenge-error-text") 
-          || message.contains("cType: 'managed'")
-          || message.contains("Just a moment...") {
-        return SoraError::CloudFlareChallenge;
-      }
-    }
     502 => {
       return SoraError::BadGateway(message);
     }
-    504 => {
-      if message.contains("cloudflare") 
-          || message.contains("Cloudflare") {
-        return SoraError::CloudFlareTimeout(message);
-      }
-    }
-    524 => {
-      // NB: 524 is a cloudflare specific gateway timeout error with slightly different semantics than 504.
-      if message.contains("cloudflare") 
-          || message.contains("Cloudflare") {
-        return SoraError::CloudFlareTimeout(message);
-      }
-    }
     _ => {}, // Fall-through
   }
 

crates/api_clients/storyteller_client/Cargo.toml

@@ -24,12 +24,13 @@ workspace-build-acceleration.workspace = true
 
 # Internal
 artcraft_api_defs.workspace = true
+cloudflare_errors.workspace = true
 enums.workspace = true
 enums_public.workspace = true
 errors.workspace = true
 idempotency.workspace = true
-tokens.workspace = true
 mimetypes.workspace = true
+tokens.workspace = true
 
 # External
 anyhow.workspace = true

crates/api_clients/storyteller_client/src/error/api_error.rs

@@ -1,7 +1,8 @@
+use cloudflare_errors::cloudflare_error::CloudflareError;
+use errors::AnyhowError;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 use std::io;
-use errors::AnyhowError;
 
 #[derive(Debug)]
 pub enum ApiError {
@@ -26,6 +27,9 @@ pub enum ApiError {
     backend_hostname: Option<String>,
   },
 
+  /// Cloudflare errors.
+  CloudflareError(CloudflareError),
+
   /// A deserialization error with the response.
   DeserializationError(serde_json::Error),
 
@@ -63,6 +67,8 @@ impl Display for ApiError {
       // Network errors
       ApiError::Timeout(msg) => write!(f, "Timeout: {}", msg),
       ApiError::NetworkError(msg) => write!(f, "Network error: {}", msg),
+      // Cloudflare errors
+      ApiError::CloudflareError(error) => write!(f, "Cloudflare Error: {}", error),
       // I/O errors
       ApiError::IoError(error) => write!(f, "IO error: {}", error),
       // Other
@@ -95,3 +101,9 @@ impl From<io::Error> for ApiError {
     ApiError::IoError(error)
   }
 }
+
+impl From<CloudflareError> for ApiError {
+  fn from(error: CloudflareError) -> Self {
+    ApiError::CloudflareError(error)
+  }
+}

crates/api_clients/storyteller_client/src/utils/filter_bad_response.rs

@@ -1,6 +1,7 @@
 use crate::error::api_error::ApiError;
 use crate::utils::status_codes::*;
 use anyhow::anyhow;
+use cloudflare_errors::filter_cloudflare_errors::filter_cloudflare_errors;
 use log::error;
 
 /// Pass request errors to this method for standard error handling behavior.
@@ -13,30 +14,34 @@ use log::error;
 pub async fn filter_bad_response(response: reqwest::Response) -> Result<reqwest::Response, ApiError> {
   let status = response.status();
 
-  if !status.is_success() {
-    // The host in the fleet that handled the request.
-    let backend_hostname= response.headers()
-        .get("x-backend-hostname")
-        .map(|header| header.to_str().unwrap_or_else(|err| {
-          error!("Failed to parse x-backend-hostname header: {:?}", err);
-          "unknown"
-        }))
-        .map(|s| s.to_string());
-    
-    let response_body = match response.text().await {
-      Ok(text) => text.to_string(),
-      Err(err) => format!("Could not read response body: {:?}", err),
-    };
-
-    return match status.as_u16() {
-      STATUS_401_UNAUTHORIZED => Err(ApiError::Unauthorized(response_body)),
-      STATUS_403_FORBIDDEN => Err(ApiError::Forbidden(response_body)),
-      STATUS_404_NOT_FOUND => Err(ApiError::NotFound(response_body)),
-      STATUS_429_TOO_MANY_REQUESTS => Err(ApiError::TooManyRequests(response_body)),
-      STATUS_500_INTERNAL_SERVER_ERROR => Err(ApiError::InternalServerError { body: response_body, backend_hostname }),
-      _ => Err(ApiError::Other(anyhow!("Bad status code: {}; message: {:?}", status, response_body))),
-    }
+  if status.is_success() {
+    return Ok(response);
   }
 
-  Ok(response)
-}
+  // The host in the fleet that handled the request.
+  let backend_hostname= response.headers()
+      .get("x-backend-hostname")
+      .map(|header| header.to_str().unwrap_or_else(|err| {
+        error!("Failed to parse x-backend-hostname header: {:?}", err);
+        "unknown"
+      }))
+      .map(|s| s.to_string());
+
+  let response_body = match response.text().await {
+    Ok(text) => text.to_string(),
+    Err(err) => format!("Could not read response body: {:?}", err),
+  };
+  
+  let status_code = status.as_u16();
+
+  filter_cloudflare_errors(status_code, &response_body)?;
+
+  match status_code {
+    STATUS_401_UNAUTHORIZED => Err(ApiError::Unauthorized(response_body)),
+    STATUS_403_FORBIDDEN => Err(ApiError::Forbidden(response_body)),
+    STATUS_404_NOT_FOUND => Err(ApiError::NotFound(response_body)),
+    STATUS_429_TOO_MANY_REQUESTS => Err(ApiError::TooManyRequests(response_body)),
+    STATUS_500_INTERNAL_SERVER_ERROR => Err(ApiError::InternalServerError { body: response_body, backend_hostname }),
+    _ => Err(ApiError::Other(anyhow!("Bad status code: {}; message: {:?}", status, response_body))),
+  }
+}

crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs

@@ -145,10 +145,12 @@ pub async fn sora_image_remix_command(
           status = CommandErrorStatus::BadRequest;
           error_message = "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
         },
-        InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => {
-          error_type = SoraImageRemixErrorType::SoraIsHavingProblems;
-          status = CommandErrorStatus::ServerError;
-          error_message = "Sora is having problems. Please try again later.";
+        InnerError::SoraError(err) => {
+          if err.is_sora_having_downtime_issues() {
+            error_type = SoraImageRemixErrorType::SoraIsHavingProblems;
+            status = CommandErrorStatus::ServerError;
+            error_message = "Sora is having problems. Please try again later.";
+          }
         }
         _ => {},
       }

crates/lib/cloudflare_errors/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "cloudflare_errors"
+edition = "2024"
+version = "0.0.1"
+publish = false
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[lib]
+name = "cloudflare_errors"
+path = "src/lib.rs"
+
+[dependencies]
+# cargo-hakari workspace hack (faster compile times)
+workspace-build-acceleration.workspace = true
+
+# Internal
+# (None)
+
+# External
+# (None)
+
+[dev-dependencies]
+# (None)

crates/lib/cloudflare_errors/src/cloudflare_error.rs

@@ -0,0 +1,32 @@
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+#[derive(Debug)]
+pub enum CloudflareError {
+  /// Cloudflare wants to verify the request with a CAPTCHA challenge.
+  ChallengeInterstitial403,
+  
+  /// Cloudflare could not form a connection to the backend server.
+  GatewayTimeout504,
+
+  /// Cloudflare formed a TCP connection to the backend server, but no payload was delivered before timeout
+  TimeoutOccurred524,
+}
+
+impl Error for CloudflareError {}
+
+impl Display for CloudflareError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+    match self {
+      Self::ChallengeInterstitial403 => {
+        write!(f, "Cloudflare Challenge Interstitial (403); Cloudflare wants to verify the request with a CAPTCHA challenge.")
+      }
+      Self::GatewayTimeout504 => {
+        write!(f, "Cloudflare Gateway Timeout (504); This is likely a backend server issue.")
+      }
+      Self::TimeoutOccurred524 => {
+        write!(f, "Cloudflare Timeout (524); This is likely a backend server issue. Cloudflare connected, but did not receive a response from the server in time.")
+      }
+    }
+  }
+}

crates/lib/cloudflare_errors/src/filter_cloudflare_errors.rs

@@ -0,0 +1,44 @@
+use crate::cloudflare_error::CloudflareError;
+
+/// This only filters out Cloudflare errors. Any other errors will be returned as `Ok(())`.
+pub fn filter_cloudflare_errors(status_code: u16, body: &str) -> Result<(), CloudflareError> {
+  if status_code >= 200 && status_code < 300 {
+    return Ok(());
+  }
+
+  match status_code {
+    403 => {
+      if body.contains("challenge-platform")
+          || body.contains("challenge-error-text")
+          || body.contains("cType: 'managed'")
+          || body.contains("Just a moment...") {
+        return Err(CloudflareError::ChallengeInterstitial403);
+      }
+    }
+    _ => {}, // Fall-through
+  }
+
+  // TODO: This is a really bad heuristic.
+  let is_cloudflare = body.contains("cloudflare")
+      || body.contains("Cloudflare");
+
+  // let is_cloudflare = body.contains("cloudflare.com")
+  //     || body.contains("Cloudflare Ray ID");
+
+  if is_cloudflare {
+    match status_code {
+      504 => return Err(CloudflareError::GatewayTimeout504),
+      524 => return Err(CloudflareError::TimeoutOccurred524),
+      _ => {}, // Fall-through
+    }
+
+    if body.contains("errorcode_504")
+        || body.contains("Gateway time-out")
+        || body.contains("Error code 504")
+    {
+      return Err(CloudflareError::GatewayTimeout504);
+    }
+  }
+
+  Ok(())
+}

crates/lib/cloudflare_errors/src/lib.rs

@@ -0,0 +1,22 @@
+//! cloudflare_errors
+//!
+//! Classify common cloudflare errors so clients can handle appropriately.
+//!
+
+// Never allow these
+#![forbid(private_bounds)]
+#![forbid(private_interfaces)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod cloudflare_error;
+pub mod filter_cloudflare_errors;

crates/schema/database/sqlite_tasks/Cargo.toml

@@ -38,9 +38,6 @@ serde.workspace = true
 serde_derive.workspace = true
 serde_json.workspace = true
 
-# For the bootstrap tool (TODO: Move this)
-tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] }
-
 [dev-dependencies]
 # serial_test = "3.0.0"