alphard/stable-diffusion.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: set eps of ggml_norm(LayerNorm) to 1e-5

Browse Source
This commit is contained in:
leejet 2023-10-27 00:50:23 +08:00
parent 4c96185fcc
commit ed374983f3
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
stable-diffusion.cpp
View File

@ -18,6 +18,8 @@
#include "rng_philox.h"
#include "stable-diffusion.h"
#define EPS 1e-05
static SDLogLevel log_level = SDLogLevel::INFO;
#define __FILENAME__ "stable-diffusion.cpp"
@ -586,7 +588,7 @@ struct ResidualAttentionBlock {
// layer norm 1
{
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx,
ggml_mul(ctx, ggml_repeat(ctx, ln1_w, x), x),
ggml_repeat(ctx, ln1_b, x));
@ -636,7 +638,7 @@ struct ResidualAttentionBlock {
// layer norm 2
{
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, ln2_w, x), x),
ggml_repeat(ctx, ln2_b, x));
@ -766,7 +768,7 @@ struct CLIPTextModel {
// final layer norm
{
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, final_ln_w, x), x),
ggml_repeat(ctx, final_ln_b, x));
@ -1200,7 +1202,7 @@ struct SpatialTransformer {
// layer norm 1
{
x = ggml_reshape_2d(ctx, x, c, w * h * n);
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, transformer.norm1_w, x),
@ -1248,7 +1250,7 @@ struct SpatialTransformer {
// layer norm 2
{
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, transformer.norm2_w, x), x),
@ -1299,7 +1301,7 @@ struct SpatialTransformer {
// layer norm 3
{
x = ggml_reshape_2d(ctx, x, c, h * w * n); // [N * h * w, in_channels]
x = ggml_norm(ctx, x, 1e-6f);
x = ggml_norm(ctx, x, EPS);
x = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, transformer.norm3_w, x), x),