fix bosta

Introduced in 2b6ec97fe2

conditioner.hpp

@@ -52,6 +52,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
     std::string trigger_word = "img";  // should be user settable
     std::string embd_dir;
     int32_t num_custom_embeddings   = 0;
+    int32_t num_custom_embeddings_2 = 0;
     std::vector<uint8_t> token_embed_custom;
     std::vector<std::string> readed_embeddings;
 
@@ -131,18 +132,31 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
         params.no_alloc               = false;
         struct ggml_context* embd_ctx = ggml_init(params);
         struct ggml_tensor* embd      = NULL;
-        int64_t hidden_size           = text_model->model.hidden_size;
+        struct ggml_tensor* embd2     = NULL;
         auto on_load                  = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) {
-            if (tensor_storage.ne[0] != hidden_size) {
+            if (tensor_storage.ne[0] != text_model->model.hidden_size) {
-                LOG_DEBUG("embedding wrong hidden size, got %i, expected %i", tensor_storage.ne[0], hidden_size);
+                if (text_model2) {
+                    if (tensor_storage.ne[0] == text_model2->model.hidden_size) {
+                        embd2       = ggml_new_tensor_2d(embd_ctx, tensor_storage.type, text_model2->model.hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
+                        *dst_tensor = embd2;
+                    } else {
+                        LOG_DEBUG("embedding wrong hidden size, got %i, expected %i or %i", tensor_storage.ne[0], text_model->model.hidden_size, text_model2->model.hidden_size);
                         return false;
                     }
-            embd        = ggml_new_tensor_2d(embd_ctx, tensor_storage.type, hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
+                } else {
+                    LOG_DEBUG("embedding wrong hidden size, got %i, expected %i", tensor_storage.ne[0], text_model->model.hidden_size);
+                    return false;
+                }
+            } else {
+                embd        = ggml_new_tensor_2d(embd_ctx, tensor_storage.type, text_model->model.hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
                 *dst_tensor = embd;
+            }
             return true;
         };
         model_loader.load_tensors(on_load, NULL);
         readed_embeddings.push_back(embd_name);
+        if (embd) {
+            int64_t hidden_size = text_model->model.hidden_size;
             token_embed_custom.resize(token_embed_custom.size() + ggml_nbytes(embd));
             memcpy((void*)(token_embed_custom.data() + num_custom_embeddings * hidden_size * ggml_type_size(embd->type)),
                    embd->data,
@@ -153,6 +167,20 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                 num_custom_embeddings++;
             }
             LOG_DEBUG("embedding '%s' applied, custom embeddings: %i", embd_name.c_str(), num_custom_embeddings);
+        }
+        if (embd2) {
+            int64_t hidden_size = text_model2->model.hidden_size;
+            token_embed_custom.resize(token_embed_custom.size() + ggml_nbytes(embd2));
+            memcpy((void*)(token_embed_custom.data() + num_custom_embeddings_2 * hidden_size * ggml_type_size(embd2->type)),
+                   embd2->data,
+                   ggml_nbytes(embd2));
+            for (int i = 0; i < embd2->ne[1]; i++) {
+                bpe_tokens.push_back(text_model2->model.vocab_size + num_custom_embeddings_2);
+                // LOG_DEBUG("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
+                num_custom_embeddings_2++;
+            }
+            LOG_DEBUG("embedding '%s' applied, custom embeddings: %i (text model 2)", embd_name.c_str(), num_custom_embeddings_2);
+        }
         return true;
     }
 

examples/cli/main.cpp

@@ -126,9 +126,9 @@ struct SDParams {
     int upscale_repeats           = 1;
 
     std::vector<int> skip_layers = {7, 8, 9};
-    float slg_scale              = 0.;
+    float slg_scale              = 0.f;
-    float skip_layer_start       = 0.01;
+    float skip_layer_start       = 0.01f;
-    float skip_layer_end         = 0.2;
+    float skip_layer_end         = 0.2f;
 };
 
 void print_params(SDParams params) {
@@ -931,12 +931,12 @@ int main(int argc, const char* argv[]) {
         }
     }
 
+    std::vector<uint8_t> default_mask_image_vec(params.width * params.height, 255);
     if (params.mask_path != "") {
         int c             = 0;
         mask_image_buffer = stbi_load(params.mask_path.c_str(), &params.width, &params.height, &c, 1);
     } else {
-        std::vector<uint8_t> arr(params.width * params.height, 255);
+        mask_image_buffer = default_mask_image_vec.data();
-        mask_image_buffer = arr.data();
     }
     sd_image_t mask_image = {(uint32_t)params.width,
                              (uint32_t)params.height,

ggml

@@ -1 +1 @@
-Subproject commit 6fcbd60bc72ac3f7ad43f78c87e535f2e6206f58
+Subproject commit ff9052988b76e137bcf92bb335733933ca196ac0

gits_noise.inl

@@ -329,21 +329,21 @@ const std::vector<std::vector<float>> GITS_NOISE_1_50 = {
 };
 
 const std::vector<const std::vector<std::vector<float>>*> GITS_NOISE = {
-    { &GITS_NOISE_0_80 },
+    &GITS_NOISE_0_80,
-    { &GITS_NOISE_0_85 },
+    &GITS_NOISE_0_85,
-    { &GITS_NOISE_0_90 },
+    &GITS_NOISE_0_90,
-    { &GITS_NOISE_0_95 },
+    &GITS_NOISE_0_95,
-    { &GITS_NOISE_1_00 },
+    &GITS_NOISE_1_00,
-    { &GITS_NOISE_1_05 },
+    &GITS_NOISE_1_05,
-    { &GITS_NOISE_1_10 },
+    &GITS_NOISE_1_10,
-    { &GITS_NOISE_1_15 },
+    &GITS_NOISE_1_15,
-    { &GITS_NOISE_1_20 },
+    &GITS_NOISE_1_20,
-    { &GITS_NOISE_1_25 },
+    &GITS_NOISE_1_25,
-    { &GITS_NOISE_1_30 },
+    &GITS_NOISE_1_30,
-    { &GITS_NOISE_1_35 },
+    &GITS_NOISE_1_35,
-    { &GITS_NOISE_1_40 },
+    &GITS_NOISE_1_40,
-    { &GITS_NOISE_1_45 },
+    &GITS_NOISE_1_45,
-    { &GITS_NOISE_1_50 }
+    &GITS_NOISE_1_50
 };
 
 #endif // GITS_NOISE_INL

model.cpp

@@ -1929,9 +1929,6 @@ bool ModelLoader::load_tensors(std::map<std::string, struct ggml_tensor*>& tenso
         if (pair.first.find("cond_stage_model.transformer.text_model.encoder.layers.23") != std::string::npos) {
             continue;
         }
-        if (pair.first.find("alphas_cumprod") != std::string::npos) {
-            continue;
-        }
 
         if (pair.first.find("alphas_cumprod") != std::string::npos) {
             continue;

model.h

@@ -14,6 +14,7 @@
 #include "ggml.h"
 #include "json.hpp"
 #include "zip.h"
+#include "gguf.h"
 
 #define SD_MAX_DIMS 5
 

stable-diffusion.cpp

@@ -1551,6 +1551,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
     }
 
     struct ggml_init_params params;
+    if (sd_version_is_sdxl(sd_ctx->sd->version)) { params.mem_size *= 4; }
     params.mem_size = static_cast<size_t>(10 * 1024 * 1024);  // 10 MB
     if (sd_version_is_sd3(sd_ctx->sd->version)) {
         params.mem_size *= 3;
@@ -1806,7 +1807,7 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
 
     size_t t2 = ggml_time_ms();
 
-    LOG_INFO("img2img completed in %.2fs", (t1 - t0) * 1.0f / 1000);
+    LOG_INFO("img2img completed in %.2fs", (t2 - t0) * 1.0f / 1000);
 
     return result_images;
 }

stable-diffusion.h

@@ -92,12 +92,15 @@ enum sd_type_t {
     SD_TYPE_F64     = 28,
     SD_TYPE_IQ1_M   = 29,
     SD_TYPE_BF16    = 30,
-    SD_TYPE_Q4_0_4_4 = 31,
+    // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
-    SD_TYPE_Q4_0_4_8 = 32,
+    // SD_TYPE_Q4_0_4_8 = 32,
-    SD_TYPE_Q4_0_8_8 = 33,
+    // SD_TYPE_Q4_0_8_8 = 33,
     SD_TYPE_TQ1_0   = 34,
     SD_TYPE_TQ2_0   = 35,
-    SD_TYPE_COUNT,
+    // SD_TYPE_IQ4_NL_4_4 = 36,
+    // SD_TYPE_IQ4_NL_4_8 = 37,
+    // SD_TYPE_IQ4_NL_8_8 = 38,
+    SD_TYPE_COUNT   = 39,
 };
 
 SD_API const char* sd_type_name(enum sd_type_t type);

tae.hpp

@@ -201,7 +201,7 @@ struct TinyAutoEncoder : public GGMLRunner {
                     bool decoder_only = true,
                     SDVersion version = VERSION_SD1)
         : decode_only(decoder_only),
-          taesd(decode_only, version),
+          taesd(decoder_only, version),
           GGMLRunner(backend) {
         taesd.init(params_ctx, tensor_types, prefix);
     }

thirdparty/stb_image_write.h

@@ -177,7 +177,7 @@ STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const
 STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void  *data);
 STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void  *data);
 STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data);
-STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void  *data, int quality);
+STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void  *data, int quality, const char* parameters = NULL);
 
 #ifdef STBIW_WINDOWS_UTF8
 STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input);