From 734d8830b69468d151deda4a504bf202f8aabd45 Mon Sep 17 00:00:00 2001
From: Aditya Vats <avats.osc@gmail.com>
Date: Tue, 9 Jan 2024 23:14:19 +0530
Subject: [PATCH] Add KL divergence loss function in machine_learning
 algorithms

---
 DIRECTORY.md                                  |  1 +
 .../loss_function/kl_divergence_loss.rs       | 37 +++++++++++++++++++
 src/machine_learning/loss_function/mod.rs     |  2 +
 src/machine_learning/mod.rs                   |  1 +
 4 files changed, 41 insertions(+)
 create mode 100644 src/machine_learning/loss_function/kl_divergence_loss.rs

diff --git a/DIRECTORY.md b/DIRECTORY.md
index abdd848d093..19b064001d7 100644
--- a/DIRECTORY.md
+++ b/DIRECTORY.md
@@ -145,6 +145,7 @@
     * Loss Function
       * [Mean Absolute Error Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/mean_absolute_error_loss.rs)
       * [Mean Squared Error Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/mean_squared_error_loss.rs)
+      * [KL Divergence Loss](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/loss_function/kl_divergence_loss.rs)
     * Optimization
       * [Adam](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/optimization/adam.rs)
       * [Gradient Descent](https://github.com/TheAlgorithms/Rust/blob/master/src/machine_learning/optimization/gradient_descent.rs)
diff --git a/src/machine_learning/loss_function/kl_divergence_loss.rs b/src/machine_learning/loss_function/kl_divergence_loss.rs
new file mode 100644
index 00000000000..f477607b20f
--- /dev/null
+++ b/src/machine_learning/loss_function/kl_divergence_loss.rs
@@ -0,0 +1,37 @@
+//! # KL divergence Loss Function
+//!
+//! For a pair of actual and predicted probability distributions represented as vectors `actual` and `predicted`, the KL divergence loss is calculated as:
+//!
+//! `L = -Σ(actual[i] * ln(predicted[i]/actual[i]))` for all `i` in the range of the vectors
+//!
+//! Where `ln` is the natural logarithm function, and `Σ` denotes the summation over all elements of the vectors.
+//!
+//! ## KL divergence Loss Function Implementation
+//!
+//! This implementation takes two references to vectors of f64 values, `actual` and `predicted`, and returns the KL divergence loss between them.
+//!
+pub fn kld_loss(actual: &[f64], predicted: &[f64]) -> f64 {
+    // epsilon to handle if any of the elements are zero
+    let eps = 0.00001f64;
+    let loss: f64 = actual
+        .iter()
+        .zip(predicted.iter())
+        .map(|(&a, &p)| ((a + eps) * ((a + eps) / (p + eps)).ln()))
+        .sum();
+    loss
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_kld_loss() {
+        let test_vector_actual = vec![1.346112, 1.337432, 1.246655];
+        let test_vector = vec![1.033836, 1.082015, 1.117323];
+        assert_eq!(
+            kld_loss(&test_vector_actual, &test_vector),
+            0.7752789394328498
+        );
+    }
+}
diff --git a/src/machine_learning/loss_function/mod.rs b/src/machine_learning/loss_function/mod.rs
index 8391143985d..a4a7d92da69 100644
--- a/src/machine_learning/loss_function/mod.rs
+++ b/src/machine_learning/loss_function/mod.rs
@@ -1,5 +1,7 @@
+mod kl_divergence_loss;
 mod mean_absolute_error_loss;
 mod mean_squared_error_loss;
 
+pub use self::kl_divergence_loss::kld_loss;
 pub use self::mean_absolute_error_loss::mae_loss;
 pub use self::mean_squared_error_loss::mse_loss;
diff --git a/src/machine_learning/mod.rs b/src/machine_learning/mod.rs
index 7277a2c9030..604019faefe 100644
--- a/src/machine_learning/mod.rs
+++ b/src/machine_learning/mod.rs
@@ -8,6 +8,7 @@ mod simpson;
 pub use self::cholesky::cholesky;
 pub use self::k_means::k_means;
 pub use self::linear_regression::linear_regression;
+pub use self::loss_function::kld_loss;
 pub use self::loss_function::mae_loss;
 pub use self::loss_function::mse_loss;
 pub use self::optimization::gradient_descent;