From 9599d01508e3d34002cade0ea7860ddb824afaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 22 Mar 2023 15:18:48 -0600 Subject: [PATCH] Implement fast_erf() and fast_gelu() This approximates the erf() function directly. --- gpt2.f90 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gpt2.f90 b/gpt2.f90 index 6f8651a..638434f 100644 --- a/gpt2.f90 +++ b/gpt2.f90 @@ -43,6 +43,21 @@ elemental real(sp) function fast_tanh(x) result(y) end if end function +elemental real function fast_erf(x) result(y) +real(sp), intent(in) :: x +real(sp) :: abs_x +abs_x = abs(x) +y = 1 - 1 / (1+ abs_x * (0.278393 + abs_x * (0.230389 + abs_x * (0.000972 + 0.078108*abs_x))))**4 +y = merge(y, -y, x >= 0) +end function + +elemental real function fast_gelu(x) result(y) +real(sp), intent(in) :: x +real(sp), parameter :: inverse_root_2 = 1 / sqrt(2._sp) +y = 0.5_sp * x * (1 + fast_erf(x * inverse_root_2)) +end function + + elemental real(sp) function gelu(x) result(y) real(sp), intent(in) :: x y = 0.5_sp * x * (1 + tanh(sqrt(2 / pi) * (x + 0.044715_sp * x**3)))