diff --git a/torchbiggraph/train.py b/torchbiggraph/train.py
index 33301fa2..b6cecbb7 100644
--- a/torchbiggraph/train.py
+++ b/torchbiggraph/train.py
@@ -470,7 +470,8 @@ def make_optimizer(params: Iterable[torch.nn.Parameter], is_emb: bool) -> Optimi
         subprocess_init=subprocess_init,
     )
     checkpoint_manager.register_metadata_provider(ConfigMetadataProvider(config))
-    checkpoint_manager.write_config(config)
+    if rank == 0:
+        checkpoint_manager.write_config(config)
 
     if config.num_edge_chunks is not None:
         num_edge_chunks = config.num_edge_chunks