fix for t5

plutonium-239 · May 15, 2024 · de68424 · de68424
1 parent 5923c23
commit de68424
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/memsave_torch/nn/LayerNorm.py b/memsave_torch/nn/LayerNorm.py
@@ -168,6 +168,8 @@ def from_existing(cls, ln):
         Returns:
             obj: The MemSaveRMSLayerNorm object
         """
+        if ln.variance_epsilon is not None:  # T5LayerNorm
+            ln.eps = ln.variance_epsilon
         obj = cls(
             ln.weight.shape,
             ln.eps,