diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index c3da036..b3896ea 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -18,6 +18,8 @@ #include "rng_philox.h" #include "stable-diffusion.h" +#define EPS 1e-05 + static SDLogLevel log_level = SDLogLevel::INFO; #define __FILENAME__ "stable-diffusion.cpp" @@ -586,7 +588,7 @@ struct ResidualAttentionBlock { // layer norm 1 { - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, ln1_w, x), x), ggml_repeat(ctx, ln1_b, x)); @@ -636,7 +638,7 @@ struct ResidualAttentionBlock { // layer norm 2 { - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, ln2_w, x), x), ggml_repeat(ctx, ln2_b, x)); @@ -766,7 +768,7 @@ struct CLIPTextModel { // final layer norm { - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, final_ln_w, x), x), ggml_repeat(ctx, final_ln_b, x)); @@ -1200,7 +1202,7 @@ struct SpatialTransformer { // layer norm 1 { x = ggml_reshape_2d(ctx, x, c, w * h * n); - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, transformer.norm1_w, x), @@ -1248,7 +1250,7 @@ struct SpatialTransformer { // layer norm 2 { - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, transformer.norm2_w, x), x), @@ -1299,7 +1301,7 @@ struct SpatialTransformer { // layer norm 3 { x = ggml_reshape_2d(ctx, x, c, h * w * n); // [N * h * w, in_channels] - x = ggml_norm(ctx, x, 1e-6f); + x = ggml_norm(ctx, x, EPS); x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, transformer.norm3_w, x), x),