diff --git a/mlx/backend/cuda/worker.cpp b/mlx/backend/cuda/worker.cpp
index de38a06e51..6266d6099b 100644
--- a/mlx/backend/cuda/worker.cpp
+++ b/mlx/backend/cuda/worker.cpp
@@ -57,14 +57,17 @@ void Worker::commit(cudaStream_t stream) {
 }
 
 void Worker::thread_fn() {
-  while (!stop_) {
-    uint64_t current_batch = 0;
+  uint64_t current_batch = 0;
+  while (true) {
     Tasks tasks;
     {
       std::unique_lock<std::mutex> lk(mtx_);
-      cond_.wait(lk, [this, &current_batch] {
+      cond_.wait(lk, [this, current_batch] {
         return this->signaled_batch_ > current_batch || this->stop_;
       });
+      if (stop_) {
+        return;
+      }
       current_batch = signaled_batch_;
       auto end = worker_tasks_.upper_bound(current_batch);
       for (auto it = worker_tasks_.begin(); it != end; ++it) {