[BE] Change _marked_safe_globals_list to set (pytorch#139303)

Prevent same global from being added multiple times

Pull Request resolved: pytorch#139303
Approved by: https://github.com/janeyx99
ghstack dependencies: pytorch#138936, pytorch#139221, pytorch#139433, pytorch#139541, pytorch#137602

Author: mikaylagawarecki

torch/_weights_only_unpickler.py

@@ -83,29 +83,27 @@
     "nt",
 ]
 
-_marked_safe_globals_list: List[Any] = []
+_marked_safe_globals_set: Set[Any] = set()
 
 
 def _add_safe_globals(safe_globals: List[Any]):
-    global _marked_safe_globals_list
-    _marked_safe_globals_list += safe_globals
+    global _marked_safe_globals_set
+    _marked_safe_globals_set = _marked_safe_globals_set.union(set(safe_globals))
 
 
 def _get_safe_globals() -> List[Any]:
-    global _marked_safe_globals_list
-    return _marked_safe_globals_list
+    global _marked_safe_globals_set
+    return list(_marked_safe_globals_set)
 
 
 def _clear_safe_globals():
-    global _marked_safe_globals_list
-    _marked_safe_globals_list = []
+    global _marked_safe_globals_set
+    _marked_safe_globals_set = set()
 
 
 def _remove_safe_globals(globals_to_remove: List[Any]):
-    global _marked_safe_globals_list
-    _marked_safe_globals_list = list(
-        set(_marked_safe_globals_list) - set(globals_to_remove)
-    )
+    global _marked_safe_globals_set
+    _marked_safe_globals_set = _marked_safe_globals_set - set(globals_to_remove)
 
 
 class _safe_globals:
@@ -128,7 +126,7 @@ def __exit__(self, type, value, tb):
 # _get_allowed_globals due to the lru_cache
 def _get_user_allowed_globals():
     rc: Dict[str, Any] = {}
-    for f in _marked_safe_globals_list:
+    for f in _marked_safe_globals_set:
         module, name = f.__module__, f.__name__
         rc[f"{module}.{name}"] = f
     return rc