diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 5affda3bc..e84a1b96c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -6,7 +6,9 @@ body: - type: markdown attributes: value: | - Please use this template and include as many details as possible to help us reproduce and fix the issue. + Before submitting a bug report, please read the [Troubleshooting guide](https://github.com/leejet/stable-diffusion.cpp/blob/master/docs/troubleshooting.md) and try the steps relevant to your problem. + + If the problem persists, complete this form and include what you tried and the results, along with enough details to help us reproduce and fix the issue. - type: textarea id: commit attributes: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..36c4fd9e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: Troubleshooting + url: https://github.com/leejet/stable-diffusion.cpp/blob/master/docs/troubleshooting.md + about: Read the troubleshooting guide first. If the problem persists, submit a bug report. diff --git a/README.md b/README.md index cb0efe2d1..d99348cbf 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ For runtime and parameter backend placement, see the [backend selection guide](. ## More Guides +- [Troubleshooting](./docs/troubleshooting.md) - [Backend selection](./docs/backend.md) - [RPC](./docs/rpc.md) - [LoRA](./docs/lora.md) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 000000000..22007db2f --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,45 @@ +# Troubleshooting + +## Completely black or white images or videos / NaNs + +Some ggml backends can encounter numerical overflow during inference, producing +NaN (not-a-number) values. This can result in completely black or white images or videos. +Whether it happens can depend on the backend, device, model, and weight format. + +Known overflow issues have been addressed as far as possible, but the maintainer +has limited hardware and cannot test every combination. Some cases may therefore +still need a manual workaround. + +These options are supported by both `sd-cli` and `sd-server`. If you encounter +this problem, add them to your CLI generation command or server startup command: + +```sh +--linear-scale 0.0078125 --attn-scale 0.0078125 +``` + +For `sd-server`, restart the server after changing these startup options. Run the +same prompt and seed again to see whether the output recovers. If the problem +persists, try smaller positive values, for example: + +```sh +--linear-scale 0.00390625 --attn-scale 0.00390625 +``` + +These options reduce intermediate values and compensate afterwards to preserve +the intended output scale: + +- `--linear-scale` scales Linear inputs before matrix multiplication and rescales + the result. +- `--attn-scale` scales attention keys and values (K/V). It takes effect only in + the Flash Attention path, where `--fa` or `--diffusion-fa` is enabled and the + backend supports it. + +The two values can be set independently and apply across model components. The +default `0` preserves each model's built-in settings; `1` explicitly disables the +corresponding scaling. Overrides must be finite positive values. C API users can +set `linear_scale` and `attn_scale` in `sd_ctx_params_t`. + +If the problem persists after trying the relevant steps above, +[submit a bug report](https://github.com/leejet/stable-diffusion.cpp/issues/new?template=bug_report.yml). +Include your full command, backend and hardware, model and weight format, logs, +and the scale values you tried with their results. diff --git a/examples/cli/README.md b/examples/cli/README.md index 9df3bb0b0..fe47122b6 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -22,3 +22,6 @@ Metadata mode inspects PNG/JPEG container metadata without loading any model: ./bin/sd-cli -M metadata --image ./output.png --metadata-raw ./bin/sd-cli -M metadata --image ./output.png --metadata-all ``` + +For completely black or white images or videos, NaNs, and the `--linear-scale` / +`--attn-scale` workaround, see [Troubleshooting](../../docs/troubleshooting.md). diff --git a/examples/common/common.cpp b/examples/common/common.cpp index dee151bfc..cdf21bc7c 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -359,6 +359,25 @@ bool parse_options(int argc, const char** argv, const std::vector& o return true; } +static int parse_scale_override(int argc, const char** argv, int index, float& scale) { + if (++index >= argc) { + return -1; + } + try { + size_t end = 0; + const std::string value = argv[index]; + float parsed = std::stof(value, &end); + if (end != value.size() || !std::isfinite(parsed) || parsed < 0.f || + (parsed > 0.f && !std::isfinite(1.f / parsed))) { + return -1; + } + scale = parsed; + } catch (const std::exception&) { + return -1; + } + return 1; +} + ArgOptions SDContextParams::get_options() { ArgOptions options; options.string_options = { @@ -687,6 +706,18 @@ ArgOptions SDContextParams::get_options() { }; options.manual_options = { + {"", + "--linear-scale", + "linear input scale override (float, default: 0 = model default, 1 = no scaling)", + [this](int argc, const char** argv, int index) { + return parse_scale_override(argc, argv, index, linear_scale); + }}, + {"", + "--attn-scale", + "flash-attention K/V scale override (float, default: 0 = model default, 1 = no scaling); requires --fa or --diffusion-fa", + [this](int argc, const char** argv, int index) { + return parse_scale_override(argc, argv, index, attn_scale); + }}, {"", "--auto-fit", "on|off (default: on). Use one GPU for diffusion/te/vae computation and place weights on that GPU, " @@ -895,6 +926,8 @@ std::string SDContextParams::to_string() const { << " vae_on_cpu: " << (vae_on_cpu ? "true" : "false") << ",\n" << " flash_attn: " << (flash_attn ? "true" : "false") << ",\n" << " diffusion_flash_attn: " << (diffusion_flash_attn ? "true" : "false") << ",\n" + << " linear_scale: " << linear_scale << ",\n" + << " attn_scale: " << attn_scale << ",\n" << " diffusion_conv_direct: " << (diffusion_conv_direct ? "true" : "false") << ",\n" << " vae_conv_direct: " << (vae_conv_direct ? "true" : "false") << ",\n" << " prediction: " << sd_prediction_name(prediction) << ",\n" @@ -948,6 +981,8 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) { sd_ctx_params.enable_mmap = enable_mmap; sd_ctx_params.flash_attn = flash_attn; sd_ctx_params.diffusion_flash_attn = diffusion_flash_attn; + sd_ctx_params.linear_scale = linear_scale; + sd_ctx_params.attn_scale = attn_scale; sd_ctx_params.tae_preview_only = taesd_preview; sd_ctx_params.diffusion_conv_direct = diffusion_conv_direct; sd_ctx_params.vae_conv_direct = vae_conv_direct; diff --git a/examples/common/common.h b/examples/common/common.h index ef084dfd4..660fef4fc 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -175,6 +175,8 @@ struct SDContextParams { lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO; bool force_sdxl_vae_conv_scale = false; + float linear_scale = 0.f; + float attn_scale = 0.f; float flow_shift = INFINITY; ArgOptions get_options(); diff --git a/examples/server/README.md b/examples/server/README.md index 7622eb407..c59fc8aeb 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -129,3 +129,6 @@ For detailed command-line arguments, run: ```bash ./bin/sd-server -h ``` + +For completely black or white images or videos, NaNs, and the `--linear-scale` / +`--attn-scale` startup options, see [Troubleshooting](../../docs/troubleshooting.md). diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 2d7d68573..cf69b48d0 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -241,6 +241,8 @@ typedef struct { const char* rpc_servers; const char* model_args; bool disable_segmented_compute; // Force monolithic graph execution even when automatic graph cutting would fit memory better + float linear_scale; // Override linear input scaling; 0 keeps the model default + float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default } sd_ctx_params_t; typedef struct { diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 019425f98..b332ba5a8 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -150,6 +150,7 @@ struct Conditioner { virtual void set_graph_cut_layer_split_backend_vram_limits(const std::vector& limits) {} virtual void get_layer_split_param_tensors(std::map& tensors) {} virtual void set_flash_attention_enabled(bool enabled) = 0; + virtual void set_scale_overrides(float linear_scale, float attn_scale) {} virtual void set_weight_adapter(const std::shared_ptr& adapter) {} virtual void runner_end() {} }; @@ -232,6 +233,13 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + text_model->set_scale_overrides(linear_scale, attn_scale); + if (sd_version_is_sdxl(version)) { + text_model2->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { text_model->set_weight_adapter(adapter); if (sd_version_is_sdxl(version)) { @@ -737,6 +745,18 @@ struct SD3CLIPEmbedder : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + if (clip_l) { + clip_l->set_scale_overrides(linear_scale, attn_scale); + } + if (clip_g) { + clip_g->set_scale_overrides(linear_scale, attn_scale); + } + if (t5) { + t5->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { if (clip_l) { clip_l->set_weight_adapter(adapter); @@ -1107,6 +1127,15 @@ struct FluxCLIPEmbedder : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + if (clip_l) { + clip_l->set_scale_overrides(linear_scale, attn_scale); + } + if (t5) { + t5->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { if (clip_l) { clip_l->set_weight_adapter(adapter); @@ -1369,6 +1398,12 @@ struct T5CLIPEmbedder : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + if (t5) { + t5->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { if (t5) { t5->set_weight_adapter(adapter); @@ -1577,6 +1612,12 @@ struct MiniT2IConditioner : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + if (t5) { + t5->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { if (t5) { t5->set_weight_adapter(adapter); @@ -1738,6 +1779,10 @@ struct AnimaConditioner : public Conditioner { llm->set_flash_attention_enabled(enabled); } + void set_scale_overrides(float linear_scale, float attn_scale) override { + llm->set_scale_overrides(linear_scale, attn_scale); + } + void set_weight_adapter(const std::shared_ptr& adapter) override { llm->set_weight_adapter(adapter); } @@ -1942,6 +1987,13 @@ struct LLMEmbedder : public Conditioner { } } + void set_scale_overrides(float linear_scale, float attn_scale) override { + llm->set_scale_overrides(linear_scale, attn_scale); + if (byt5) { + byt5->set_scale_overrides(linear_scale, attn_scale); + } + } + void set_weight_adapter(const std::shared_ptr& adapter) override { if (llm) { llm->set_weight_adapter(adapter); @@ -3031,6 +3083,11 @@ struct LTXAVEmbedder : public Conditioner { projector->set_flash_attention_enabled(enabled); } + void set_scale_overrides(float linear_scale, float attn_scale) override { + llm->set_scale_overrides(linear_scale, attn_scale); + projector->set_scale_overrides(linear_scale, attn_scale); + } + void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); projector->set_max_graph_vram_bytes(max_vram_bytes); diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index 3737dc7aa..57b4c4825 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -2,6 +2,7 @@ #include #include +#include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" #include "core/ggml_runner.h" #include "core/ggml_tensor_utils.h" @@ -11,6 +12,21 @@ using namespace sd; +ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx, + ggml_tensor* q, + ggml_tensor* k, + ggml_tensor* v, + int64_t n_head, + ggml_tensor* mask, + bool skip_reshape, + bool flash_attn, + float kv_scale) { + if (ctx->attn_scale > 0.f) { + kv_scale = ctx->attn_scale; + } + return ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, skip_reshape, flash_attn, kv_scale); +} + void GGMLRunner::alloc_params_ctx() { ggml_init_params params; params.mem_size = static_cast(MAX_PARAMS_TENSOR_NUM * ggml_tensor_overhead()); @@ -510,6 +526,8 @@ GGMLRunnerContext GGMLRunner::get_context() { runner_ctx.ggml_ctx = compute_ctx; runner_ctx.backend = runtime_backend; runner_ctx.flash_attn_enabled = flash_attn_enabled; + runner_ctx.linear_scale = linear_scale; + runner_ctx.attn_scale = attn_scale; runner_ctx.conv2d_direct_enabled = conv2d_direct_enabled; runner_ctx.circular_x_enabled = circular_x_enabled; runner_ctx.circular_y_enabled = circular_y_enabled; diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h index 7960f5aaa..5dc3fa5a1 100644 --- a/src/core/ggml_runner.h +++ b/src/core/ggml_runner.h @@ -68,6 +68,8 @@ struct GGMLRunnerContext { ggml_backend_t backend = nullptr; ggml_context* ggml_ctx = nullptr; bool flash_attn_enabled = false; + float linear_scale = 0.f; + float attn_scale = 0.f; bool conv2d_direct_enabled = false; bool circular_x_enabled = false; bool circular_y_enabled = false; @@ -113,6 +115,16 @@ struct GGMLRunnerContext { } }; +ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx, + ggml_tensor* q, + ggml_tensor* k, + ggml_tensor* v, + int64_t n_head, + ggml_tensor* mask = nullptr, + bool skip_reshape = false, + bool flash_attn = false, + float kv_scale = 1.f); + struct GGMLRunner { private: std::map logged_compute_bytes_; @@ -163,6 +175,8 @@ struct GGMLRunner { const std::string final_result_name = "ggml_runner_final_result_tensor"; bool flash_attn_enabled = false; + float linear_scale = 0.f; + float attn_scale = 0.f; bool conv2d_direct_enabled = false; bool circular_x_enabled = false; bool circular_y_enabled = false; @@ -323,6 +337,11 @@ struct GGMLRunner { flash_attn_enabled = enabled; } + void set_scale_overrides(float linear_scale, float attn_scale) { + this->linear_scale = linear_scale; + this->attn_scale = attn_scale; + } + void set_conv2d_direct_enabled(bool enabled) { conv2d_direct_enabled = enabled; } diff --git a/src/extensions/photomaker_extension.cpp b/src/extensions/photomaker_extension.cpp index 05bb46c0d..a15e751e3 100644 --- a/src/extensions/photomaker_extension.cpp +++ b/src/extensions/photomaker_extension.cpp @@ -135,6 +135,7 @@ struct PhotoMakerExtension : public GenerationExtension { pm_version, 20.f, ctx.model_manager); + pmid_model->set_scale_overrides(ctx.params->linear_scale, ctx.params->attn_scale); if (pm_version == PM_VERSION_2) { LOG_INFO("using PhotoMaker Version 2"); } diff --git a/src/model/adapter/ip_adapter.hpp b/src/model/adapter/ip_adapter.hpp index 2cc660b59..9f8dc9f64 100644 --- a/src/model/adapter/ip_adapter.hpp +++ b/src/model/adapter/ip_adapter.hpp @@ -95,7 +95,7 @@ namespace IPAdapter { int64_t L = kv->ne[1]; ggml_tensor* k = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0)); ggml_tensor* v = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0])); - ggml_tensor* attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false); + ggml_tensor* attn = ggml_ext_attention_ext(ctx, q, k, v, heads, nullptr, false, false); attn = to_out->forward(ctx, attn); latents = ggml_add(ctx->ggml_ctx, latents, attn); diff --git a/src/model/adapter/pulid.hpp b/src/model/adapter/pulid.hpp index 65b3064a2..3ba84fb6d 100644 --- a/src/model/adapter/pulid.hpp +++ b/src/model/adapter/pulid.hpp @@ -63,12 +63,11 @@ class PuLIDPerceiverAttentionCA : public GGMLBlock { k = ggml_cont(ctx->ggml_ctx, k); v = ggml_cont(ctx->ggml_ctx, v); - ggml_tensor* attn_out = ggml_ext_attention_ext( - ctx->ggml_ctx, ctx->backend, - q, k, v, - heads, - /*mask=*/nullptr, - /*diag_mask_inf=*/false); + ggml_tensor* attn_out = ggml_ext_attention_ext(ctx, + q, k, v, + heads, + /*mask=*/nullptr, + /*diag_mask_inf=*/false); ggml_tensor* out = to_out->forward(ctx, attn_out); return out; diff --git a/src/model/common/block.hpp b/src/model/common/block.hpp index 09db9250a..5a5e6615c 100644 --- a/src/model/common/block.hpp +++ b/src/model/common/block.hpp @@ -380,14 +380,14 @@ class CrossAttention : public GGMLBlock { if (xtra_dim) { context->ne[0] = 320; // reset dim to orig } - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, inner_dim] + x = ggml_ext_attention_ext(ctx, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, inner_dim] if (has_ip && ctx->ip_context != nullptr && ctx->ip_scale != 0.0f) { auto to_k_ip = std::dynamic_pointer_cast(blocks["to_k_ip"]); auto to_v_ip = std::dynamic_pointer_cast(blocks["to_v_ip"]); auto k_ip = to_k_ip->forward(ctx, ctx->ip_context); auto v_ip = to_v_ip->forward(ctx, ctx->ip_context); - auto x_ip = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k_ip, v_ip, n_head, nullptr, false, ctx->flash_attn_enabled); + auto x_ip = ggml_ext_attention_ext(ctx, q, k_ip, v_ip, n_head, nullptr, false, ctx->flash_attn_enabled); x = ggml_add(ctx->ggml_ctx, x, ggml_scale(ctx->ggml_ctx, x_ip, ctx->ip_scale)); } diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index 0b3cb8650..cc5470c34 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -206,6 +206,7 @@ class Linear : public UnaryBlock { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { ggml_tensor* w = params["weight"]; + const float scale = ctx->linear_scale > 0.f ? ctx->linear_scale : this->scale; ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr; if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) { bool supports_fp8_matmul = false; @@ -870,7 +871,7 @@ class MultiheadAttention : public GGMLBlock { v = v_proj->forward(ctx, x); } - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, false); // [N, n_token, embed_dim] + x = ggml_ext_attention_ext(ctx, q, k, v, n_head, mask, false); // [N, n_token, embed_dim] x = out_proj->forward(ctx, x); // [N, n_token, embed_dim] return x; diff --git a/src/model/common/rope.hpp b/src/model/common/rope.hpp index 4f2580394..447c3a23e 100644 --- a/src/model/common/rope.hpp +++ b/src/model/common/rope.hpp @@ -1024,7 +1024,7 @@ namespace Rope { q = apply_rope(ctx->ggml_ctx, q, pe, rope_interleaved); // [N*n_head, L, d_head] k = apply_rope(ctx->ggml_ctx, k, pe, rope_interleaved); // [N*n_head, L, d_head] - auto x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, true, ctx->flash_attn_enabled, kv_scale); // [N, L, n_head*d_head] + auto x = ggml_ext_attention_ext(ctx, q, k, v, n_head, mask, true, ctx->flash_attn_enabled, kv_scale); // [N, L, n_head*d_head] return x; } }; // namespace Rope diff --git a/src/model/diffusion/anima.hpp b/src/model/diffusion/anima.hpp index 930efacea..a8977c966 100644 --- a/src/model/diffusion/anima.hpp +++ b/src/model/diffusion/anima.hpp @@ -237,8 +237,7 @@ namespace Anima { } auto q_rope = Rope::apply_rope(ctx->ggml_ctx, q4, pe_q, false); auto k_rope = Rope::apply_rope(ctx->ggml_ctx, k4, pe_k, false); - attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + attn_out = ggml_ext_attention_ext(ctx, q_rope, k_rope, v4, @@ -249,8 +248,7 @@ namespace Anima { } else { auto q_flat = ggml_reshape_3d(ctx->ggml_ctx, q4, head_dim * num_heads, L_q, N); auto k_flat = ggml_reshape_3d(ctx->ggml_ctx, k4, head_dim * num_heads, L_k, N); - attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + attn_out = ggml_ext_attention_ext(ctx, q_flat, k_flat, v, diff --git a/src/model/diffusion/animatediff.hpp b/src/model/diffusion/animatediff.hpp index 6a94e21df..eb0ffba31 100644 --- a/src/model/diffusion/animatediff.hpp +++ b/src/model/diffusion/animatediff.hpp @@ -61,7 +61,7 @@ namespace AnimateDiff { auto k = to_k->forward(ctx, x_pe); auto v = to_v->forward(ctx, x_pe); - auto a = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, (int)num_heads, nullptr, false); + auto a = ggml_ext_attention_ext(ctx, q, k, v, (int)num_heads, nullptr, false); return to_out->forward(ctx, a); } }; diff --git a/src/model/diffusion/ernie_image.hpp b/src/model/diffusion/ernie_image.hpp index c4a5df192..81bf142a0 100644 --- a/src/model/diffusion/ernie_image.hpp +++ b/src/model/diffusion/ernie_image.hpp @@ -183,7 +183,7 @@ namespace ErnieImage { k = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, k, 0, 2, 1, 3)); // [N, heads, S, head_dim] k = ggml_reshape_3d(ctx->ggml_ctx, k, k->ne[0], k->ne[1], k->ne[2] * k->ne[3]); - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, attention_mask, true, ctx->flash_attn_enabled); // [N, S, hidden_size] + x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, ctx->flash_attn_enabled); // [N, S, hidden_size] x = to_out_0->forward(ctx, x); return x; } diff --git a/src/model/diffusion/hidream_o1.hpp b/src/model/diffusion/hidream_o1.hpp index 677f81e7d..9e5a901f3 100644 --- a/src/model/diffusion/hidream_o1.hpp +++ b/src/model/diffusion/hidream_o1.hpp @@ -504,6 +504,10 @@ namespace HiDreamO1 { vision_runner->set_flash_attention_enabled(enabled); } + void set_scale_overrides(float linear_scale, float attn_scale) override { + vision_runner->set_scale_overrides(linear_scale, attn_scale); + } + void set_weight_adapter(const std::shared_ptr& adapter) override { vision_runner->set_weight_adapter(adapter); } diff --git a/src/model/diffusion/hunyuan.hpp b/src/model/diffusion/hunyuan.hpp index c111674f5..c106b9cd5 100644 --- a/src/model/diffusion/hunyuan.hpp +++ b/src/model/diffusion/hunyuan.hpp @@ -54,7 +54,7 @@ namespace Hunyuan { auto k = qkv_vec[1]; auto v = qkv_vec[2]; - auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, mask, false, ctx->flash_attn_enabled); + auto attn_out = ggml_ext_attention_ext(ctx, q, k, v, num_heads, mask, false, ctx->flash_attn_enabled); attn_out = self_attn_proj->forward(ctx, attn_out); // adaLN_modulation diff --git a/src/model/diffusion/krea2.hpp b/src/model/diffusion/krea2.hpp index 41760581d..84ecfe0b1 100644 --- a/src/model/diffusion/krea2.hpp +++ b/src/model/diffusion/krea2.hpp @@ -232,8 +232,7 @@ namespace Krea2 { q = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, q), head_dim_ * heads, Lq, N); k = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, k), head_dim_ * kv_heads, Lk, N); v = ggml_reshape_3d(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, v), head_dim_ * kv_heads, Lk, N); - return ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + return ggml_ext_attention_ext(ctx, q, k, v, diff --git a/src/model/diffusion/ltxv.hpp b/src/model/diffusion/ltxv.hpp index 118606e4c..31a65afa6 100644 --- a/src/model/diffusion/ltxv.hpp +++ b/src/model/diffusion/ltxv.hpp @@ -709,8 +709,7 @@ namespace LTXV { k = apply_hidden_rope(ctx->ggml_ctx, k, k_pe, heads, dim_head, rope_interleaved); } - auto out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + auto out = ggml_ext_attention_ext(ctx, q, k, v, diff --git a/src/model/diffusion/minimax_h3.hpp b/src/model/diffusion/minimax_h3.hpp index 3e2b69db0..80c29ab52 100644 --- a/src/model/diffusion/minimax_h3.hpp +++ b/src/model/diffusion/minimax_h3.hpp @@ -215,8 +215,7 @@ namespace MiniMaxH3 { q = attention_layout(ctx->ggml_ctx, q); k = attention_layout(ctx->ggml_ctx, k); } - auto out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + auto out = ggml_ext_attention_ext(ctx, q, k, v, diff --git a/src/model/diffusion/mmdit.hpp b/src/model/diffusion/mmdit.hpp index 454ef4b3e..42599c67a 100644 --- a/src/model/diffusion/mmdit.hpp +++ b/src/model/diffusion/mmdit.hpp @@ -365,8 +365,8 @@ class SelfAttention : public GGMLBlock { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto qkv = pre_attention(ctx, x); - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] - x = post_attention(ctx, x); // [N, n_token, dim] + x = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + x = post_attention(ctx, x); // [N, n_token, dim] return x; } }; @@ -587,8 +587,8 @@ struct DismantledBlock : public GGMLBlock { auto qkv2 = std::get<1>(qkv_intermediates); auto intermediates = std::get<2>(qkv_intermediates); - auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] - auto attn2_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn2_out = ggml_ext_attention_ext(ctx, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] x = post_attention_x(ctx, attn_out, attn2_out, @@ -604,7 +604,7 @@ struct DismantledBlock : public GGMLBlock { auto qkv = qkv_intermediates.first; auto intermediates = qkv_intermediates.second; - auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] x = post_attention(ctx, attn_out, intermediates[0], @@ -648,7 +648,7 @@ block_mixing(GGMLRunnerContext* ctx, qkv.push_back(ggml_concat(ctx->ggml_ctx, context_qkv[i], x_qkv[i], 1)); } - auto attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_context + n_token, hidden_size] + auto attn = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_context + n_token, hidden_size] auto context_attn = ggml_view_3d(ctx->ggml_ctx, attn, @@ -680,7 +680,7 @@ block_mixing(GGMLRunnerContext* ctx, } if (x_block->self_attn) { - auto attn2 = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, hidden_size] + auto attn2 = ggml_ext_attention_ext(ctx, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, hidden_size] x = x_block->post_attention_x(ctx, x_attn, diff --git a/src/model/diffusion/wan.hpp b/src/model/diffusion/wan.hpp index 687f28380..363beeeb2 100644 --- a/src/model/diffusion/wan.hpp +++ b/src/model/diffusion/wan.hpp @@ -193,7 +193,7 @@ namespace WAN { k = norm_k->forward(ctx, k); auto v = v_proj->forward(ctx, context); // [N, n_context, dim] - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] x = o_proj->forward(ctx, x); // [N, n_token, dim] return x; @@ -255,8 +255,8 @@ namespace WAN { k_img = norm_k_img->forward(ctx, k_img); auto v_img = v_img_proj->forward(ctx, context_img); // [N, context_img_len, dim] - auto img_x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k_img, v_img, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto img_x = ggml_ext_attention_ext(ctx, q, k_img, v_img, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] x = ggml_add(ctx->ggml_ctx, x, img_x); diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index 4f1db3bca..de01e5de8 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -1359,7 +1359,7 @@ namespace LLM { x = ggml_ext_cont(ctx->ggml_ctx, kqv); x = ggml_reshape_3d(ctx->ggml_ctx, x, head_dim * num_heads, n_token, N); } else { - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, attention_mask, true, false); // [N, n_token, hidden_size] + x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, false); // [N, n_token, hidden_size] } x = out_proj->forward(ctx, x); // [N, n_token, hidden_size] diff --git a/src/model/te/t5.hpp b/src/model/te/t5.hpp index f89488d36..adc68aa09 100644 --- a/src/model/te/t5.hpp +++ b/src/model/te/t5.hpp @@ -251,7 +251,7 @@ class T5Attention : public GGMLBlock { k = ggml_ext_scale(ctx->ggml_ctx, k, ::sqrtf(static_cast(d_head)), true); - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, mask); // [N, n_token, d_head * n_head] + x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, mask); // [N, n_token, d_head * n_head] x = out_proj->forward(ctx, x); // [N, n_token, model_dim] return {x, past_bias}; diff --git a/src/model/vae/auto_encoder_kl.hpp b/src/model/vae/auto_encoder_kl.hpp index 116a83219..115ff3458 100644 --- a/src/model/vae/auto_encoder_kl.hpp +++ b/src/model/vae/auto_encoder_kl.hpp @@ -142,7 +142,7 @@ class AttnBlock : public UnaryBlock { v = ggml_reshape_3d(ctx->ggml_ctx, v, c, h * w, n); // [N, h * w, in_channels] } - h_ = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); + h_ = ggml_ext_attention_ext(ctx, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); if (use_linear) { h_ = proj_out->forward(ctx, h_); // [N, h * w, in_channels] diff --git a/src/model/vae/hunyuan_vae.hpp b/src/model/vae/hunyuan_vae.hpp index 099a15b7e..1aab8ac53 100644 --- a/src/model/vae/hunyuan_vae.hpp +++ b/src/model/vae/hunyuan_vae.hpp @@ -193,7 +193,7 @@ namespace Hunyuan { v = ggml_reshape_3d(ctx->ggml_ctx, v, w * h * t, c, b); // [b, c, t*h*w] v = ggml_ext_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, v, 1, 0, 2, 3)); // [b, t*h*w, c] - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); // [b, t*h*w, c] + x = ggml_ext_attention_ext(ctx, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); // [b, t*h*w, c] x = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [b, c, t*h*w] x = ggml_reshape_4d(ctx->ggml_ctx, x, w, h, t, c * b); // [b*c, t, h, w] diff --git a/src/model/vae/mage_vae.hpp b/src/model/vae/mage_vae.hpp index 5567aec22..38ff89c4f 100644 --- a/src/model/vae/mage_vae.hpp +++ b/src/model/vae/mage_vae.hpp @@ -253,7 +253,7 @@ namespace MageVAE { q = to_patches(ctx->ggml_ctx, q); k = to_patches(ctx->ggml_ctx, k); v = to_patches(ctx->ggml_ctx, v); - h = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); + h = ggml_ext_attention_ext(ctx, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); h = from_patches(ctx->ggml_ctx, h, np, batch, hp, wp); if (pad_h > 0) { h = ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, height); diff --git a/src/model/vae/minimax_h3_audio_vae.hpp b/src/model/vae/minimax_h3_audio_vae.hpp index 926d2fbe5..a4fc14473 100644 --- a/src/model/vae/minimax_h3_audio_vae.hpp +++ b/src/model/vae/minimax_h3_audio_vae.hpp @@ -174,8 +174,7 @@ namespace MiniMaxH3 { auto mask = ggml_diag_mask_inf(ctx->ggml_ctx, ggml_ext_zeros(ctx->ggml_ctx, sequence, sequence, 1, 1), 0); - auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + auto attn_out = ggml_ext_attention_ext(ctx, q, k, v, diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index ec65fa37e..777566438 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -291,8 +291,7 @@ namespace MiniMaxH3VAE { k = ggml_rms_norm(ctx->ggml_ctx, k, 1e-5f); q = apply_partial_rope(ctx->ggml_ctx, q, pe); k = apply_partial_rope(ctx->ggml_ctx, k, pe); - auto out = ggml_ext_attention_ext(ctx->ggml_ctx, - ctx->backend, + auto out = ggml_ext_attention_ext(ctx, q, k, v, diff --git a/src/model/vae/wan_vae.hpp b/src/model/vae/wan_vae.hpp index a632d66b4..a56392e81 100644 --- a/src/model/vae/wan_vae.hpp +++ b/src/model/vae/wan_vae.hpp @@ -615,8 +615,8 @@ namespace WAN { auto v = qkv_vec[2]; v = ggml_reshape_3d(ctx->ggml_ctx, v, h * w, c, n); // [t, c, h * w] - v = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, v, 1, 0, 2, 3)); // [t, h * w, c] - x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); // [t, h * w, c] + v = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, v, 1, 0, 2, 3)); // [t, h * w, c] + x = ggml_ext_attention_ext(ctx, q, k, v, 1, nullptr, false, ctx->flash_attn_enabled); // [t, h * w, c] x = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [t, c, h * w] x = ggml_reshape_4d(ctx->ggml_ctx, x, w, h, c, n); // [t, c, h, w] diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 2ae1431df..5d51e2417 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -847,6 +847,12 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf } bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) { + for (float scale : {sd_ctx_params->linear_scale, sd_ctx_params->attn_scale}) { + if (!std::isfinite(scale) || scale < 0.f || (scale > 0.f && !std::isfinite(1.f / scale))) { + LOG_ERROR("scale overrides must be finite positive values, or 0 to keep model defaults"); + return false; + } + } auto configuration = std::make_unique(*sd_ctx_params); n_threads = sd_ctx_params->n_threads; enable_mmap = sd_ctx_params->enable_mmap; diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index 24f2fbc00..da5a469e8 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -403,6 +403,21 @@ namespace sd::model_builders { "ip_adapter", weight_manager); } + if (result.conditioner) { + result.conditioner->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.diffusion) { + result.diffusion->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.high_noise_diffusion) { + result.high_noise_diffusion->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.clip_vision) { + result.clip_vision->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.ip_adapter) { + result.ip_adapter->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } runners = std::move(result); return true; } @@ -538,6 +553,15 @@ namespace sd::model_builders { result.preview->set_conv2d_direct_enabled(true); } } + if (result.vae) { + result.vae->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.preview) { + result.preview->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } + if (result.audio) { + result.audio->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); + } runners = std::move(result); return true; } @@ -559,6 +583,7 @@ namespace sd::model_builders { LOG_INFO("Using Conv2d direct in the control net"); control_net->set_conv2d_direct_enabled(true); } + control_net->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale); runner = std::move(control_net); return true; } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index efb22b0d4..beb899cdb 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -323,6 +323,8 @@ void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { sd_ctx_params->eager_load = false; sd_ctx_params->enable_mmap = false; sd_ctx_params->diffusion_flash_attn = false; + sd_ctx_params->linear_scale = 0.f; + sd_ctx_params->attn_scale = 0.f; sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO; sd_ctx_params->backend = nullptr; sd_ctx_params->params_backend = nullptr; @@ -374,6 +376,8 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { "auto_fit: %s\n" "flash_attn: %s\n" "diffusion_flash_attn: %s\n" + "linear_scale: %g\n" + "attn_scale: %g\n" "vae_format: %s\n", SAFE_STR(sd_ctx_params->model_path), SAFE_STR(sd_ctx_params->clip_l_path), @@ -409,6 +413,8 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { BOOL_STR(sd_ctx_params->auto_fit), BOOL_STR(sd_ctx_params->flash_attn), BOOL_STR(sd_ctx_params->diffusion_flash_attn), + sd_ctx_params->linear_scale, + sd_ctx_params->attn_scale, sd_vae_format_name(sd_ctx_params->vae_format)); return buf;