diff --git a/pipegoose/distributed/parallel_context.py b/pipegoose/distributed/parallel_context.py
index 59ce580..19d3c88 100644
--- a/pipegoose/distributed/parallel_context.py
+++ b/pipegoose/distributed/parallel_context.py
@@ -329,6 +329,8 @@ def destroy(self):
 
         dist.barrier()
         dist.destroy_process_group()
-        rpc.shutdown()
+
+        if self.get_world_size(ParallelMode.GLOBAL) > 1:
+            rpc.shutdown()
 
         self._groups.clear()