From 34f2ecbc3bc0ae8ba0666808db7de19fb3d907d4 Mon Sep 17 00:00:00 2001
From: Laurent Mazare <laurent.mazare@gmail.com>
Date: Tue, 19 Sep 2023 18:17:17 +0100
Subject: [PATCH] Fix the leaky relu. (#898)

---
 candle-nn/src/ops.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/candle-nn/src/ops.rs b/candle-nn/src/ops.rs
index 1256a076..32de1af9 100644
--- a/candle-nn/src/ops.rs
+++ b/candle-nn/src/ops.rs
@@ -45,7 +45,8 @@ pub fn sigmoid(xs: &Tensor) -> Result<Tensor> {
 }
 
 pub fn leaky_relu(xs: &Tensor, negative_slope: f64) -> Result<Tensor> {
-    xs.relu()?.minimum(&(xs * negative_slope)?)
+    let zeros = xs.zeros_like()?;
+    xs.maximum(&zeros)? + xs.minimum(&zeros)? * negative_slope
 }
 
 pub fn dropout(xs: &Tensor, drop_p: f32) -> Result<Tensor> {