Introduce two new job status types to generic inference jobs

_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/down.sql

@@ -0,0 +1,13 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    MODIFY status ENUM(
+        'pending',
+        'started',
+        'complete_success',
+        'complete_failure',
+        'attempt_failed',
+        'dead'
+  ) NOT NULL DEFAULT 'pending';

_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/up.sql

@@ -0,0 +1,16 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Add two states to allow us to distinguish user-cancelled jobs and system-cancelled jobs.
+ALTER TABLE generic_inference_jobs
+    MODIFY status ENUM(
+        'pending',
+        'started',
+        'complete_success',
+        'complete_failure',
+        'attempt_failed',
+        'dead',
+        'cancelled_by_user',
+        'cancelled_by_system'
+) NOT NULL DEFAULT 'pending';

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -145,9 +145,11 @@ CREATE TABLE generic_inference_jobs (
   --  * Pending = job is ready to go
   --  * Started = job is running
   --  * Complete_Success = job is done (success)
-  --  * Complete_Failure = job is done (failure)
+  --  * Complete_Failure = job is done (permanent failure, eg. a Face Animator job without a face)
   --  * Attempt_Failed = job failed but may retry.
-  --  * Dead = job failed permanently.
+  --  * Dead = job failed permanently (ie. exhausted retries)
+  --  * Cancelled_By_User = the user canceled their own job
+  --  * Cancelled_By_System = the system terminated the job (ie. load shedding, user account deletion, etc.)
   --
   -- Pending -> Started -> Complete_Success
   --                    |-> Complete_Failure
@@ -158,7 +160,10 @@ CREATE TABLE generic_inference_jobs (
     'complete_success',
     'complete_failure',
     'attempt_failed',
-    'dead') NOT NULL DEFAULT 'pending',
+    'dead',
+    'cancelled_by_user',
+    'cancelled_by_system'
+  ) NOT NULL DEFAULT 'pending',
 
   -- We can track this against a "max_attempt_count"
   attempt_count SMALLINT UNSIGNED NOT NULL DEFAULT 0,

crates/_types/enums/src/common/job_status.rs

@@ -0,0 +1,147 @@
+use std::collections::BTreeSet;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// This is used in nearly every job system as an actual MySQL ENUM value:
+///
+///  - tts_download_job
+///  - tts_inference_job
+///  - w2l_download_job
+///  - w2l_inference_job
+///  - generic_download_job
+///  - (NOT generic_inference_job, which uses JobStatusPlus)
+///
+/// See the documentation on the table for usage.
+///
+/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
+
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, sqlx::Type)]
+#[serde(rename_all = "snake_case")]
+#[sqlx(rename_all = "snake_case")]
+pub enum JobStatus {
+  Pending,
+  Started,
+  CompleteSuccess,
+  CompleteFailure,
+  AttemptFailed,
+  Dead,
+}
+
+// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
+impl_enum_display_and_debug_using_to_str!(JobStatus);
+
+// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders`
+
+impl JobStatus {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::Pending => "pending",
+      Self::Started => "started",
+      Self::CompleteSuccess => "complete_success",
+      Self::CompleteFailure => "complete_failure",
+      Self::AttemptFailed => "attempt_failed",
+      Self::Dead => "dead",
+    }
+  }
+
+  pub fn from_str(job_status: &str) -> Result<Self, String> {
+    match job_status {
+      "pending" => Ok(Self::Pending),
+      "started" => Ok(Self::Started),
+      "complete_success" => Ok(Self::CompleteSuccess),
+      "complete_failure" => Ok(Self::CompleteFailure),
+      "attempt_failed" => Ok(Self::AttemptFailed),
+      "dead" => Ok(Self::Dead),
+      _ => Err(format!("invalid job_status: {:?}", job_status)),
+    }
+  }
+
+  pub fn all_variants() -> BTreeSet<Self> {
+    // NB: BTreeSet is sorted
+    // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
+    BTreeSet::from([
+      Self::Pending,
+      Self::Started,
+      Self::CompleteSuccess,
+      Self::CompleteFailure,
+      Self::AttemptFailed,
+      Self::Dead,
+    ])
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::common::job_status::JobStatus;
+  use crate::test_helpers::assert_serialization;
+
+  mod explicit_checks {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(JobStatus::Pending, "pending");
+      assert_serialization(JobStatus::Started, "started");
+      assert_serialization(JobStatus::CompleteSuccess, "complete_success");
+      assert_serialization(JobStatus::CompleteFailure, "complete_failure");
+      assert_serialization(JobStatus::AttemptFailed, "attempt_failed");
+      assert_serialization(JobStatus::Dead, "dead");
+    }
+
+    #[test]
+    fn to_str() {
+      assert_eq!(JobStatus::Pending.to_str(), "pending");
+      assert_eq!(JobStatus::Started.to_str(), "started");
+      assert_eq!(JobStatus::CompleteSuccess.to_str(), "complete_success");
+      assert_eq!(JobStatus::CompleteFailure.to_str(), "complete_failure");
+      assert_eq!(JobStatus::AttemptFailed.to_str(), "attempt_failed");
+      assert_eq!(JobStatus::Dead.to_str(), "dead");
+    }
+
+    #[test]
+    fn from_str() {
+      assert_eq!(JobStatus::from_str("pending").unwrap(), JobStatus::Pending);
+      assert_eq!(JobStatus::from_str("started").unwrap(), JobStatus::Started);
+      assert_eq!(JobStatus::from_str("complete_success").unwrap(), JobStatus::CompleteSuccess);
+      assert_eq!(JobStatus::from_str("complete_failure").unwrap(), JobStatus::CompleteFailure);
+      assert_eq!(JobStatus::from_str("attempt_failed").unwrap(), JobStatus::AttemptFailed);
+      assert_eq!(JobStatus::from_str("dead").unwrap(), JobStatus::Dead);
+    }
+
+    #[test]
+    fn all_variants() {
+      let mut variants = JobStatus::all_variants();
+      assert_eq!(variants.len(), 6);
+      assert_eq!(variants.pop_first(), Some(JobStatus::Pending));
+      assert_eq!(variants.pop_first(), Some(JobStatus::Started));
+      assert_eq!(variants.pop_first(), Some(JobStatus::CompleteSuccess));
+      assert_eq!(variants.pop_first(), Some(JobStatus::CompleteFailure));
+      assert_eq!(variants.pop_first(), Some(JobStatus::AttemptFailed));
+      assert_eq!(variants.pop_first(), Some(JobStatus::Dead));
+      assert_eq!(variants.pop_first(), None);
+    }
+  }
+
+  mod mechanical_checks {
+    use super::*;
+
+    #[test]
+    fn variant_length() {
+      use strum::IntoEnumIterator;
+      assert_eq!(JobStatus::all_variants().len(), JobStatus::iter().len());
+    }
+
+    #[test]
+    fn round_trip() {
+      for variant in JobStatus::all_variants() {
+        assert_eq!(variant, JobStatus::from_str(variant.to_str()).unwrap());
+        assert_eq!(variant, JobStatus::from_str(&format!("{}", variant)).unwrap());
+        assert_eq!(variant, JobStatus::from_str(&format!("{:?}", variant)).unwrap());
+      }
+    }
+  }
+}

crates/_types/enums/src/common/job_status_plus.rs

@@ -0,0 +1,160 @@
+use std::collections::BTreeSet;
+
+#[cfg(test)]
+use strum::EnumCount;
+#[cfg(test)]
+use strum::EnumIter;
+
+/// This is used in newer jobs (that add additional enum states)
+///
+///  - generic_inference_job
+///  - (no other jobs yet)
+///
+/// See the documentation on the table for usage.
+///
+/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
+
+#[cfg_attr(test, derive(EnumIter, EnumCount))]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, sqlx::Type)]
+#[serde(rename_all = "snake_case")]
+#[sqlx(rename_all = "snake_case")]
+pub enum JobStatusPlus {
+  Pending,
+  Started,
+  CompleteSuccess,
+  CompleteFailure,
+  AttemptFailed,
+  Dead,
+  CancelledByUser,
+  CancelledBySystem,
+}
+
+// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
+impl_enum_display_and_debug_using_to_str!(JobStatusPlus);
+
+// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders`
+
+impl JobStatusPlus {
+  pub fn to_str(&self) -> &'static str {
+    match self {
+      Self::Pending => "pending",
+      Self::Started => "started",
+      Self::CompleteSuccess => "complete_success",
+      Self::CompleteFailure => "complete_failure",
+      Self::AttemptFailed => "attempt_failed",
+      Self::Dead => "dead",
+      Self::CancelledByUser => "cancelled_by_user",
+      Self::CancelledBySystem => "cancelled_by_system",
+    }
+  }
+
+  pub fn from_str(job_status: &str) -> Result<Self, String> {
+    match job_status {
+      "pending" => Ok(Self::Pending),
+      "started" => Ok(Self::Started),
+      "complete_success" => Ok(Self::CompleteSuccess),
+      "complete_failure" => Ok(Self::CompleteFailure),
+      "attempt_failed" => Ok(Self::AttemptFailed),
+      "dead" => Ok(Self::Dead),
+      "cancelled_by_user" => Ok(Self::CancelledByUser),
+      "cancelled_by_system" => Ok(Self::CancelledBySystem),
+      _ => Err(format!("invalid job_status: {:?}", job_status)),
+    }
+  }
+
+  pub fn all_variants() -> BTreeSet<Self> {
+    // NB: BTreeSet is sorted
+    // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
+    BTreeSet::from([
+      Self::Pending,
+      Self::Started,
+      Self::CompleteSuccess,
+      Self::CompleteFailure,
+      Self::AttemptFailed,
+      Self::Dead,
+      Self::CancelledByUser,
+      Self::CancelledBySystem,
+    ])
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::common::job_status_plus::JobStatusPlus;
+  use crate::test_helpers::assert_serialization;
+
+  mod explicit_checks {
+    use super::*;
+
+    #[test]
+    fn test_serialization() {
+      assert_serialization(JobStatusPlus::Pending, "pending");
+      assert_serialization(JobStatusPlus::Started, "started");
+      assert_serialization(JobStatusPlus::CompleteSuccess, "complete_success");
+      assert_serialization(JobStatusPlus::CompleteFailure, "complete_failure");
+      assert_serialization(JobStatusPlus::AttemptFailed, "attempt_failed");
+      assert_serialization(JobStatusPlus::Dead, "dead");
+      assert_serialization(JobStatusPlus::CancelledByUser, "cancelled_by_user");
+      assert_serialization(JobStatusPlus::CancelledBySystem, "cancelled_by_system");
+    }
+
+    #[test]
+    fn to_str() {
+      assert_eq!(JobStatusPlus::Pending.to_str(), "pending");
+      assert_eq!(JobStatusPlus::Started.to_str(), "started");
+      assert_eq!(JobStatusPlus::CompleteSuccess.to_str(), "complete_success");
+      assert_eq!(JobStatusPlus::CompleteFailure.to_str(), "complete_failure");
+      assert_eq!(JobStatusPlus::AttemptFailed.to_str(), "attempt_failed");
+      assert_eq!(JobStatusPlus::Dead.to_str(), "dead");
+      assert_eq!(JobStatusPlus::CancelledByUser.to_str(), "cancelled_by_user");
+      assert_eq!(JobStatusPlus::CancelledBySystem.to_str(), "cancelled_by_system");
+    }
+
+    #[test]
+    fn from_str() {
+      assert_eq!(JobStatusPlus::from_str("pending").unwrap(), JobStatusPlus::Pending);
+      assert_eq!(JobStatusPlus::from_str("started").unwrap(), JobStatusPlus::Started);
+      assert_eq!(JobStatusPlus::from_str("complete_success").unwrap(), JobStatusPlus::CompleteSuccess);
+      assert_eq!(JobStatusPlus::from_str("complete_failure").unwrap(), JobStatusPlus::CompleteFailure);
+      assert_eq!(JobStatusPlus::from_str("attempt_failed").unwrap(), JobStatusPlus::AttemptFailed);
+      assert_eq!(JobStatusPlus::from_str("dead").unwrap(), JobStatusPlus::Dead);
+      assert_eq!(JobStatusPlus::from_str("cancelled_by_user").unwrap(), JobStatusPlus::CancelledByUser);
+      assert_eq!(JobStatusPlus::from_str("cancelled_by_system").unwrap(), JobStatusPlus::CancelledBySystem);
+    }
+
+    #[test]
+    fn all_variants() {
+      let mut variants = JobStatusPlus::all_variants();
+      assert_eq!(variants.len(), 8);
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::Pending));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::Started));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::CompleteSuccess));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::CompleteFailure));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::AttemptFailed));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::Dead));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::CancelledByUser));
+      assert_eq!(variants.pop_first(), Some(JobStatusPlus::CancelledBySystem));
+      assert_eq!(variants.pop_first(), None);
+    }
+  }
+
+  mod mechanical_checks {
+    use super::*;
+
+    #[test]
+    fn variant_length() {
+      use strum::IntoEnumIterator;
+      assert_eq!(JobStatusPlus::all_variants().len(), JobStatusPlus::iter().len());
+    }
+
+    #[test]
+    fn round_trip() {
+      for variant in JobStatusPlus::all_variants() {
+        // Test to_str(), from_str(), Display, and Debug.
+        assert_eq!(variant, JobStatusPlus::from_str(variant.to_str()).unwrap());
+        assert_eq!(variant, JobStatusPlus::from_str(&format!("{}", variant)).unwrap());
+        assert_eq!(variant, JobStatusPlus::from_str(&format!("{:?}", variant)).unwrap());
+      }
+    }
+  }
+}

crates/_types/enums/src/common/mod.rs

@@ -1,3 +1,5 @@
+pub mod job_status;
+pub mod job_status_plus;
 pub mod sqlite;
 pub mod visibility;
 pub mod vocoder_type;

crates/lib/mysql_queries/src/column_types/job_status.rs

@@ -1,45 +0,0 @@
-//! These are columns where users can control the visibility of their data.
-
-use anyhow::anyhow;
-
-use errors::AnyhowResult;
-
-/// To use this in a query, the query must have type annotations.
-/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375
-/// eg. job_status as `job_status: crate::column_types::job_status::JobStatus`
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)]
-#[serde(rename_all = "snake_case")]
-#[sqlx(rename_all = "snake_case")]
-pub enum JobStatus {
-  Pending,
-  Started,
-  CompleteSuccess,
-  CompleteFailure,
-  AttemptFailed,
-  Dead,
-}
-
-impl JobStatus {
-  pub fn to_str(&self) -> &'static str {
-    match self {
-      Self::Pending => "pending",
-      Self::Started => "started",
-      Self::CompleteSuccess => "complete_success",
-      Self::CompleteFailure => "complete_failure",
-      Self::AttemptFailed => "attempt_failed",
-      Self::Dead => "dead",
-    }
-  }
-
-  pub fn from_str(job_status: &str) -> AnyhowResult<Self> {
-    match job_status {
-      "pending" => Ok(Self::Pending),
-      "started" => Ok(Self::Started),
-      "complete_success" => Ok(Self::CompleteSuccess),
-      "complete_failure" => Ok(Self::CompleteFailure),
-      "attempt_failed" => Ok(Self::AttemptFailed),
-      "dead" => Ok(Self::Dead),
-      _ => Err(anyhow!("invalid job_status: {:?}", job_status)),
-    }
-  }
-}

crates/lib/mysql_queries/src/column_types/mod.rs

@@ -1,3 +1,2 @@
-pub mod job_status;
 pub mod twitch_event_category;
 pub mod vocoder_type;

crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs

@@ -4,11 +4,11 @@ use chrono::Utc;
 use sqlx::MySqlPool;
 
 use enums::by_table::generic_download_jobs::generic_download_type::GenericDownloadType;
+use enums::common::job_status::JobStatus;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::generic_download_jobs::DownloadJobToken;
 
-use crate::column_types::job_status::JobStatus;
 use crate::queries::generic_download::job::_keys::GenericDownloadJobId;
 
 /// table: generic_download_jobs

crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs

@@ -6,11 +6,11 @@ use sqlx::MySqlPool;
 
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+use enums::common::job_status_plus::JobStatusPlus;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::generic_inference_jobs::InferenceJobToken;
 
-use crate::column_types::job_status::JobStatus;
 use crate::helpers::boolean_converters::i8_to_bool;
 use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
 use crate::queries::generic_inference::job::_keys::GenericInferenceJobId;
@@ -42,7 +42,7 @@ pub struct AvailableInferenceJob {
   pub creator_set_visibility: Visibility,
 
   // Job information
-  pub status: JobStatus,
+  pub status: JobStatusPlus,
   pub attempt_count: u16,
 
   pub priority_level: u16,
@@ -128,7 +128,8 @@ pub async fn list_available_generic_inference_jobs(
               .map(|args| GenericInferenceArgs::from_json(args))
               .transpose()?,
           maybe_raw_inference_text: record.maybe_raw_inference_text,
-          status: JobStatus::from_str(&record.status)?,
+          status: JobStatusPlus::from_str(&record.status)
+              .map_err(|err| anyhow!("JobStatus failure to parse: {:?}", err))?,
           attempt_count: record.attempt_count,
           priority_level: record.priority_level,
           is_keepalive_required: i8_to_bool(record.is_keepalive_required),

crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_pending_and_grab_lock.rs

@@ -62,6 +62,8 @@ FOR UPDATE
     "complete_success" => false, // Job already complete
     "complete_failure" => false, // Job already complete (permanently dead; no need to retry)
     "dead" => false, // Job already complete (permanently dead; retries exhausted)
+    "cancelled_by_user" => false, // Job already complete (permanently dead; killed by user)
+    "cancelled_by_system" => false, // Job already complete (permanently dead; killed by system)
     _ => false, // Future-proof
   };
 

crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs

@@ -1,4 +1,3 @@
 pub mod get_inference_job_status;
 pub mod get_pending_inference_job_count;
 pub mod insert_generic_inference_job;
-pub mod result_details;

crates/lib/mysql_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs

@@ -6,10 +6,10 @@
 use chrono::{DateTime, Utc};
 use sqlx::MySqlPool;
 
+use enums::common::job_status::JobStatus;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 
-use crate::column_types::job_status::JobStatus;
 use crate::helpers::boolean_converters::i8_to_bool;
 use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId;
 
@@ -83,7 +83,7 @@ SELECT
   is_for_twitch,
   is_debug_request,
 
-  status as `status: crate::column_types::job_status::JobStatus`,
+  status as `status: enums::common::job_status::JobStatus`,
   priority_level,
   attempt_count,
   failure_reason,
@@ -132,7 +132,7 @@ SELECT
   is_for_twitch,
   is_debug_request,
 
-  status as `status: crate::column_types::job_status::JobStatus`,
+  status as `status: enums::common::job_status::JobStatus`,
   priority_level,
   attempt_count,
   failure_reason,
@@ -224,7 +224,7 @@ SELECT
   is_for_twitch,
   is_debug_request,
 
-  status as `status: crate::column_types::job_status::JobStatus`,
+  status as `status: enums::common::job_status::JobStatus`,
   priority_level,
   attempt_count,
   failure_reason,