Skip to content

Commit 22213a1

Browse files
committed
Change RMSNorm eps to 1e-6 (#173)
I think this is what is used in the Python code
1 parent d7def1a commit 22213a1

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

ggml.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5556,7 +5556,7 @@ static void ggml_compute_forward_rms_norm_f32(
55565556
const size_t nb2 = dst->nb[2];
55575557
const size_t nb3 = dst->nb[3];
55585558

5559-
const ggml_float eps = 1e-5f; // TODO: make this a parameter
5559+
const ggml_float eps = 1e-6f; // TODO: make this a parameter
55605560

55615561
// TODO: optimize
55625562
for (int i03 = 0; i03 < ne03; i03++) {
@@ -5572,7 +5572,7 @@ static void ggml_compute_forward_rms_norm_f32(
55725572
mean /= ne00;
55735573

55745574
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
5575-
5575+
55765576
memcpy(y, x, ne00 * sizeof(float));
55775577
// for (int i00 = 0; i00 < ne00; i00++) {
55785578
// y[i00] = x[i00];

0 commit comments

Comments
 (0)