fix value residual

lucidrains · Oct 31, 2024 · d88537d · d88537d
1 parent 9917a88
commit d88537d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.42.2',
+  version = '1.42.3',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py
@@ -1371,7 +1371,7 @@ def forward(
                 diff_values = repeat(diff_values, 'b h n d -> b (r h) n d', r = h // kv_h)
             else:
                 # https://arxiv.org/abs/2410.17897v1
-                v = v + value_residual
+                v = 0.5 * (v + value_residual)
 
         # attention is all we need