Add RMS norm and use it (#187)

* add ggml_rms_norm

* update op num
This commit is contained in:
hoangmit
2023-03-16 00:41:38 +02:00
committed by GitHub
parent 27944c4206
commit 6eac39ba95
3 changed files with 134 additions and 5 deletions
+5
View File
@@ -230,6 +230,7 @@ enum ggml_op {
GGML_OP_GELU,
GGML_OP_SILU,
GGML_OP_NORM, // normalize
GGML_OP_RMS_NORM,
GGML_OP_MUL_MAT,
@@ -482,6 +483,10 @@ struct ggml_tensor * ggml_norm(
struct ggml_context * ctx,
struct ggml_tensor * a);
struct ggml_tensor * ggml_rms_norm(
struct ggml_context * ctx,
struct ggml_tensor * a);
// A: m rows, n columns
// B: p rows, n columns (i.e. we transpose it internally)
// result is m columns, p rows