From a12addfc5db7e3032edfc5dc3e08a1a1069f0ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 22 Sep 2023 00:59:23 +0300 Subject: [PATCH 01/23] WIP: refactoring for GGUF support --- clip.cpp | 138 ++++++++++++++++++++++++ models/convert_hf_to_gguf.py | 202 +++++++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 models/convert_hf_to_gguf.py diff --git a/clip.cpp b/clip.cpp index e94b037..b2b8945 100644 --- a/clip.cpp +++ b/clip.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,22 @@ // #define CLIP_DEBUG +int get_key_idx(const gguf_context * ctx, char * key) { + int i = gguf_find_key(ctx, key); + if (i == -1) { + fprintf(stderr, "key %s not found in file\n", key); + throw std::runtime_error("Missing required key"); + } + + return i; +} + +const int get_int(const gguf_context * ctx, char * key) { + const int i = get_key_idx(ctx, key); + + return gguf_get_val_i32(ctx, i); +} + // // Vocab utils // @@ -118,6 +135,8 @@ struct clip_buffer { }; struct clip_ctx { + bool has_text_encoder = false; + bool has_vision_encoder = false; struct clip_text_model text_model; struct clip_vision_model vision_model; struct clip_vocab vocab; @@ -127,6 +146,123 @@ struct clip_ctx { struct clip_buffer buf_compute; }; +// read and create ggml_context containing the tensors and their data +static bool load_gguf(const std::string & fname) { + + clip_ctx * new_clip = new clip_ctx; + auto & vision_model = new_clip->vision_model; + auto & ctx_data = new_clip->ctx; + + struct gguf_init_params params = { + /*.no_alloc = */ false, + /*.ctx = */ &ctx_data, + }; + + struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); + + printf("%s: version: %d\n", __func__, gguf_get_version(ctx)); + printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); + printf("%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); + + // kv + { + const int n_kv = gguf_get_n_kv(ctx); + + printf("%s: n_kv: %d\n", __func__, n_kv); + + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(ctx, i); + + printf("%s: kv[%d]: key = %s\n", __func__, i, key); + } + } + + // data + { + const int n_tensors = gguf_get_n_tensors(ctx); + + for (int i = 0; i < 3; ++i) { + const char * name = gguf_get_tensor_name(ctx, i); + + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); + + printf("%s: tensor[%d]: n_dims = %d, name = %s, data = %p\n", __func__, i, cur->n_dims, cur->name, cur->data); + } + } + + // model capabilities + { + int idx = get_key_idx(ctx, "clip.has_text_encoder"); + new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx); + + idx = get_key_idx(ctx, "clip.has_vision_encoder"); + new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx); + + printf("%s: has_text_encoder: %d\n", __func__, new_clip->has_text_encoder); + printf("%s: has_vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); + } + + printf("%s: ctx_data size (MB): %f\n", __func__, ggml_get_mem_size(ctx_data) / (1024.0 * 1024.0)); + + // text model + if (new_clip->has_text_encoder) { + // load text model + auto & text_model = new_clip->text_model; + auto & hparams = text_model.hparams; + hparams.hidden_size = get_int(ctx, "clip.text.embedding_length"); + hparams.n_head = get_int(ctx, "clip.text.attention.head_count"); + hparams.n_intermediate = get_int(ctx, "clip.text.feed_forward_length"); + hparams.n_layer = get_int(ctx, "clip.text.block_count"); + hparams.num_positions = get_int(ctx, "clip.text.context_length"); + hparams.projection_dim = get_int(ctx, "clip.text.projection_dim"); + + const int idx_tokens = get_key_idx(ctx, "tokenizer.ggml.tokens"); + hparams.n_vocab = gguf_get_arr_n(ctx, idx_tokens); + + if (1 /*verbosity >= 2*/) { + printf("\n%s: text model hparams\n", __func__); + printf("n_vocab %d\n", hparams.n_vocab); + printf("num_positions %d\n", hparams.num_positions); + printf("t_hidden_size %d\n", hparams.hidden_size); + printf("t_n_intermediate %d\n", hparams.n_intermediate); + printf("t_projection_dim %d\n", hparams.projection_dim); + printf("t_n_head %d\n", hparams.n_head); + printf("t_n_layer %d\n", hparams.n_layer); + } + } + + // vision model + if (new_clip->has_vision_encoder) { + // load vision model + auto & vision_model = new_clip->vision_model; + auto & hparams = vision_model.hparams; + hparams.hidden_size = get_int(ctx, "clip.vision.embedding_length"); + hparams.n_head = get_int(ctx, "clip.vision.attention.head_count"); + hparams.n_intermediate = get_int(ctx, "clip.vision.feed_forward_length"); + hparams.n_layer = get_int(ctx, "clip.vision.block_count"); + hparams.image_size = get_int(ctx, "clip.vision.image_size"); + hparams.patch_size = get_int(ctx, "clip.vision.patch_size"); + hparams.projection_dim = get_int(ctx, "clip.vision.projection_dim"); + + if (1 /*verbosity >= 2*/) { + + printf("\n%s: vision model hparams\n", __func__); + printf("image_size %d\n", hparams.image_size); + printf("patch_size %d\n", hparams.patch_size); + printf("v_hidden_size %d\n", hparams.hidden_size); + printf("v_n_intermediate %d\n", hparams.n_intermediate); + printf("v_projection_dim %d\n", hparams.projection_dim); + printf("v_n_head %d\n", hparams.n_head); + printf("v_n_layer %d\n", hparams.n_layer); + } + } + + ggml_free(ctx_data); + gguf_free(ctx); + + return true; +} + // utility function for a workaround until https://github.com/ggerganov/ggml/issues/260 is resolved // after that, remove this and use the mechanism implemented in GGML directly size_t get_mem_req_by_size(const size_t n_tensors, const int n_image_positions) { @@ -412,6 +548,8 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { printf("%s: loading model from '%s' - please wait...", __func__, fname); } + load_gguf(fname); + exit(0); auto fin = std::ifstream(fname, std::ios::binary); if (!fin) { diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py new file mode 100644 index 0000000..ac554fe --- /dev/null +++ b/models/convert_hf_to_gguf.py @@ -0,0 +1,202 @@ +import argparse +import os +import json + +import numpy as np +from gguf import * +from transformers import CLIPModel, CLIPProcessor + +TEXT = "clip.text" +VISION = "clip.vision" + +def k(raw_key: str, arch: str) -> str: + return raw_key.format(arch=arch) + +def should_skip_tensor(name: str, text_only: bool, vision_only: bool) -> bool: + if name in ( + "logit_scale", + "text_model.embeddings.position_ids", + "vision_model.embeddings.position_ids", + ): + return True + + if text_only and name.startswith("v"): + return True + + if vision_only and name.startswith("t"): + return True + + return False + +def get_tensor_name(name: str) -> str: + if "projection" in name: + return name + + return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "fc_up").replace("mlp.fc2", "fc_down") + + +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +ap = argparse.ArgumentParser(prog="convert_hf_to_gguf.py") +ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True) +ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16") +ap.add_argument("--text-only", action="store_true", required=False, help="Save a text-only model. It can't be used to encode images") +ap.add_argument("--vision-only", action="store_true", required=False, help="Save a vision-only model. It can't be used to encode texts") +ap.add_argument("--image-mean", nargs=3, type=float, required=False, help="Override image mean values") +ap.add_argument("--image-std", nargs=3, type=float, required=False, help="Override image std values") +args = ap.parse_args() + +if args.text_only and args.vision_only: + print("--text-only and --image-only arguments cannot be specified at the same time.") + exit(1) + +if args.use_f32: + print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.") + +# output in the same directory as the model +dir_model = args.model_dir + + +with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f: + vocab = json.load(f) + tokens = [key for key in vocab] + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + config = json.load(f) + v_hparams = config["vision_config"] + t_hparams = config["text_config"] + +# possible data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if args.use_f32: + ftype = 0 + + +model = CLIPModel.from_pretrained(dir_model) +list_vars = model.state_dict() +processor = CLIPProcessor.from_pretrained(dir_model) + +fname_middle = None +has_text_encoder = True +has_vision_encoder = True +if args.text_only: + fname_middle = "text-" + has_vision_encoder = False +elif args.vision_only: + fname_middle = "vision-" + has_text_encoder = False +else: + fname_middle = "" + +fname_out = os.path.join(dir_model, f"ggml-{fname_middle}model-{ftype_str[ftype]}.gguf") +fout = GGUFWriter(path=fname_out, arch="clip") + +fout.add_bool("clip.has_text_encoder", has_text_encoder) +fout.add_bool("clip.has_vision_encoder", has_vision_encoder) + +if has_text_encoder: + # text_model hparams + fout.add_int32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"]) + fout.add_int32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"]) + fout.add_int32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"]) + fout.add_int32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"])) + fout.add_int32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"]) + fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"]) + fout.add_int32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"]) + fout.add_token_list(tokens) + +if has_vision_encoder: + # vision_model hparams + fout.add_int32("clip.vision.image_size", v_hparams["image_size"]) + fout.add_int32("clip.vision.patch_size", v_hparams["patch_size"]) + fout.add_int32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"]) + fout.add_int32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"]) + fout.add_int32("clip.vision.projection_dim", v_hparams.get("projection_dim", config["projection_dim"])) + fout.add_int32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"]) + fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"]) + fout.add_int32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"]) + + image_mean = processor.image_processor.image_mean if args.image_mean is None else args.image_mean + image_std = processor.image_processor.image_std if args.image_std is None else args.image_std + fout.add_array("clip.vision.image_mean", image_mean) + fout.add_array("clip.vision.image_std", image_std) + +use_gelu = int(v_hparams["hidden_act"] == "gelu") +fout.add_bool("clip.use_gelu", use_gelu) + + + + +for name, data in list_vars.items(): + if should_skip_tensor(name, args.text_only, args.vision_only): + # we don't need this + print(f"skipping parameter: {name}") + continue + + name = get_tensor_name(name) + data = data.squeeze().numpy() + + n_dims = len(data.shape) + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0 + if n_dims == 4: + print(f"tensor {name} is always saved in f16") + data = data.astype(np.float16) + ftype_cur = 1 + elif ftype == 1: + if name[-7:] == ".weight" and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype_cur = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + + print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}") + fout.add_tensor(name, data) + + + +fout.write_header_to_file() +fout.write_kv_data_to_file() +fout.write_ti_data_to_file() +fout.write_tensors_to_file() +fout.close() + +print("Done. Output file: " + fname_out) \ No newline at end of file From 99f879ecd579432b0f5b74f80eed3a91697684c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 24 Sep 2023 01:48:06 +0300 Subject: [PATCH 02/23] inference with GGUF is working --- clip.cpp | 202 +++++++++++++++++++++++++++++++---- clip.h | 1 + models/convert_hf_to_gguf.py | 3 +- 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/clip.cpp b/clip.cpp index b2b8945..eb7071f 100644 --- a/clip.cpp +++ b/clip.cpp @@ -17,13 +17,45 @@ #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" -// #define CLIP_DEBUG +#define CLIP_DEBUG + +static std::string format(const char * fmt, ...) { + va_list ap; + va_list ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int size = vsnprintf(NULL, 0, fmt, ap); + GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT + std::vector buf(size + 1); + int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); + GGML_ASSERT(size2 == size); + va_end(ap2); + va_end(ap); + return std::string(buf.data(), buf.size()); +} + +#define TN_TOKEN_EMBD "%s.token_embd.weight" +#define TN_POS_EMBD "%s.position_embd.weight" +#define TN_CLASS_EMBD "v.class_embd" +#define TN_PATCH_EMBD "v.patch_embd.weight" +#define TN_ATTN_K "%s.blk.%d.attn_k.%s" +#define TN_ATTN_Q "%s.blk.%d.attn_q.%s" +#define TN_ATTN_V "%s.blk.%d.attn_v.%s" +#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s" +#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s" +#define TN_FFN_UP "%s.blk.%d.ffn_up.%s" +#define TN_LN_1 "%s.blk.%d.ln1.%s" +#define TN_LN_2 "%s.blk.%d.ln2.%s" +#define TN_LN_PRE "%s.pre_ln.%s" +#define TN_LN_POST "%s.post_ln.%s" +#define TN_TEXT_PROJ "text_projection.weight" +#define TN_VIS_PROJ "visual_projection.weight" int get_key_idx(const gguf_context * ctx, char * key) { int i = gguf_find_key(ctx, key); if (i == -1) { - fprintf(stderr, "key %s not found in file\n", key); - throw std::runtime_error("Missing required key"); + // fprintf(stderr, "key %s not found in file\n", key); + throw std::runtime_error(format("Missing required key: %s", key)); } return i; @@ -35,6 +67,16 @@ const int get_int(const gguf_context * ctx, char * key) { return gguf_get_val_i32(ctx, i); } +struct ggml_tensor * get_tensor(struct ggml_context * ctx, std::string name) { + struct ggml_tensor * cur = ggml_get_tensor(ctx, name.c_str()); + if (!cur) { + printf("unable to find tensor %s\n", name.c_str()); + throw std::runtime_error(format("unable to find tensor %s\n", name.c_str())); + } + + return cur; +} + // // Vocab utils // @@ -147,25 +189,23 @@ struct clip_ctx { }; // read and create ggml_context containing the tensors and their data -static bool load_gguf(const std::string & fname) { +struct clip_ctx * clip_model_load_gguf(const char * fname, const int verbosity = 1) { - clip_ctx * new_clip = new clip_ctx; - auto & vision_model = new_clip->vision_model; - auto & ctx_data = new_clip->ctx; + struct ggml_context * meta = NULL; struct gguf_init_params params = { - /*.no_alloc = */ false, - /*.ctx = */ &ctx_data, + /*.no_alloc = */ true, + /*.ctx = */ &meta, }; - struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); + struct gguf_context * ctx = gguf_init_from_file(fname, params); printf("%s: version: %d\n", __func__, gguf_get_version(ctx)); printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); printf("%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); // kv - { + if (verbosity >= 2) { const int n_kv = gguf_get_n_kv(ctx); printf("%s: n_kv: %d\n", __func__, n_kv); @@ -178,18 +218,33 @@ static bool load_gguf(const std::string & fname) { } // data + size_t ctx_size = 0; { const int n_tensors = gguf_get_n_tensors(ctx); - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < n_tensors; ++i) { const char * name = gguf_get_tensor_name(ctx, i); + const size_t offset = gguf_get_tensor_offset(ctx, i); + + struct ggml_tensor * cur = ggml_get_tensor(meta, name); + ctx_size += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE; + size_t tensor_size = ggml_nbytes(cur); + size_t padded_size = ggml_nbytes_pad(cur); + ctx_size += padded_size; + if (verbosity >= 2) { + printf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, padded_size=%zu, offset=%zu\n", __func__, i, + cur->n_dims, cur->name, tensor_size, padded_size, offset); + } + } - struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); - - printf("%s: tensor[%d]: n_dims = %d, name = %s, data = %p\n", __func__, i, cur->n_dims, cur->name, cur->data); + if (verbosity >= 1) { + printf("%s: ctx_meta size: %zu\n", __func__, ggml_get_mem_size(meta)); + printf("%s: ctx_data size: %zu (%f MB)\n", __func__, ctx_size, (ctx_size / (1024.0f * 1024.0f))); } } + clip_ctx * new_clip = new clip_ctx; + // model capabilities { int idx = get_key_idx(ctx, "clip.has_text_encoder"); @@ -202,7 +257,47 @@ static bool load_gguf(const std::string & fname) { printf("%s: has_vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); } - printf("%s: ctx_data size (MB): %f\n", __func__, ggml_get_mem_size(ctx_data) / (1024.0 * 1024.0)); + // load tensors + { + struct ggml_init_params params = { + .mem_size = ctx_size, + .mem_buffer = NULL, + .no_alloc = false, + }; + + new_clip->ctx = ggml_init(params); + if (!new_clip->ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + clip_free(new_clip); + return nullptr; + } + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + printf("cannot open model file for loading tensors\n"); + clip_free(new_clip); + return nullptr; + } + + const int n_tensors = gguf_get_n_tensors(ctx); + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx, i); + struct ggml_tensor * t = ggml_get_tensor(meta, name); + struct ggml_tensor * cur = ggml_dup_tensor(new_clip->ctx, t); + ggml_set_name(cur, name); + + const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i); + fin.seekg(offset, std::ios::beg); + if (!fin) { + printf("%s: failed to seek for tensor %s\n", __func__, name); + clip_free(new_clip); + return nullptr; + } + fin.read(reinterpret_cast(cur->data), ggml_nbytes(t)); + } + + fin.close(); + } // text model if (new_clip->has_text_encoder) { @@ -218,8 +313,14 @@ static bool load_gguf(const std::string & fname) { const int idx_tokens = get_key_idx(ctx, "tokenizer.ggml.tokens"); hparams.n_vocab = gguf_get_arr_n(ctx, idx_tokens); + auto & vocab = new_clip->vocab; + for (int id = 0; id < hparams.n_vocab; ++id) { + const std::string token = gguf_get_arr_str(ctx, idx_tokens, id); + vocab.id_to_token[id] = token; + vocab.token_to_id[token] = id; + } - if (1 /*verbosity >= 2*/) { + if (verbosity >= 2) { printf("\n%s: text model hparams\n", __func__); printf("n_vocab %d\n", hparams.n_vocab); printf("num_positions %d\n", hparams.num_positions); @@ -229,8 +330,36 @@ static bool load_gguf(const std::string & fname) { printf("t_n_head %d\n", hparams.n_head); printf("t_n_layer %d\n", hparams.n_layer); } + + text_model.token_embeddings = get_tensor(new_clip->ctx, format(TN_TOKEN_EMBD, "t")); + text_model.position_embeddings = get_tensor(new_clip->ctx, format(TN_POS_EMBD, "t")); + text_model.post_ln_w = get_tensor(new_clip->ctx, format(TN_LN_POST, "t", "weight")); + text_model.post_ln_b = get_tensor(new_clip->ctx, format(TN_LN_POST, "t", "bias")); + text_model.projection = get_tensor(new_clip->ctx, TN_TEXT_PROJ); + text_model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = text_model.layers[il]; + layer.k_w = get_tensor(new_clip->ctx, format(TN_ATTN_K, "t", il, "weight")); + layer.q_w = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "t", il, "weight")); + layer.v_w = get_tensor(new_clip->ctx, format(TN_ATTN_V, "t", il, "weight")); + layer.o_w = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "t", il, "weight")); + layer.ln_1_w = get_tensor(new_clip->ctx, format(TN_LN_1, "t", il, "weight")); + layer.ln_2_w = get_tensor(new_clip->ctx, format(TN_LN_2, "t", il, "weight")); + layer.ff_i_w = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "t", il, "weight")); + layer.ff_o_w = get_tensor(new_clip->ctx, format(TN_FFN_UP, "t", il, "weight")); + layer.k_b = get_tensor(new_clip->ctx, format(TN_ATTN_K, "t", il, "bias")); + layer.q_b = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "t", il, "bias")); + layer.v_b = get_tensor(new_clip->ctx, format(TN_ATTN_V, "t", il, "bias")); + layer.o_b = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "t", il, "bias")); + layer.ln_1_b = get_tensor(new_clip->ctx, format(TN_LN_1, "t", il, "bias")); + layer.ln_2_b = get_tensor(new_clip->ctx, format(TN_LN_2, "t", il, "bias")); + layer.ff_i_b = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "t", il, "bias")); + layer.ff_o_b = get_tensor(new_clip->ctx, format(TN_FFN_UP, "t", il, "bias")); + } } + new_clip->buf_compute.resize(24 * 1024 * 1024); + // vision model if (new_clip->has_vision_encoder) { // load vision model @@ -244,7 +373,7 @@ static bool load_gguf(const std::string & fname) { hparams.patch_size = get_int(ctx, "clip.vision.patch_size"); hparams.projection_dim = get_int(ctx, "clip.vision.projection_dim"); - if (1 /*verbosity >= 2*/) { + if (verbosity >= 2) { printf("\n%s: vision model hparams\n", __func__); printf("image_size %d\n", hparams.image_size); @@ -255,18 +384,48 @@ static bool load_gguf(const std::string & fname) { printf("v_n_head %d\n", hparams.n_head); printf("v_n_layer %d\n", hparams.n_layer); } + + vision_model.patch_embeddings = get_tensor(new_clip->ctx, TN_PATCH_EMBD); + vision_model.class_embedding = get_tensor(new_clip->ctx, TN_CLASS_EMBD); + vision_model.position_embeddings = get_tensor(new_clip->ctx, format(TN_POS_EMBD, "v")); + vision_model.pre_ln_w = get_tensor(new_clip->ctx, format(TN_LN_PRE, "v", "weight")); + vision_model.pre_ln_b = get_tensor(new_clip->ctx, format(TN_LN_PRE, "v", "bias")); + vision_model.post_ln_w = get_tensor(new_clip->ctx, format(TN_LN_POST, "v", "weight")); + vision_model.post_ln_b = get_tensor(new_clip->ctx, format(TN_LN_POST, "v", "bias")); + vision_model.projection = get_tensor(new_clip->ctx, TN_TEXT_PROJ); + vision_model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = vision_model.layers[il]; + layer.k_w = get_tensor(new_clip->ctx, format(TN_ATTN_K, "v", il, "weight")); + layer.q_w = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "v", il, "weight")); + layer.v_w = get_tensor(new_clip->ctx, format(TN_ATTN_V, "v", il, "weight")); + layer.o_w = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "v", il, "weight")); + layer.ln_1_w = get_tensor(new_clip->ctx, format(TN_LN_1, "v", il, "weight")); + layer.ln_2_w = get_tensor(new_clip->ctx, format(TN_LN_2, "v", il, "weight")); + layer.ff_i_w = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "v", il, "weight")); + layer.ff_o_w = get_tensor(new_clip->ctx, format(TN_FFN_UP, "v", il, "weight")); + layer.k_b = get_tensor(new_clip->ctx, format(TN_ATTN_K, "v", il, "bias")); + layer.q_b = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "v", il, "bias")); + layer.v_b = get_tensor(new_clip->ctx, format(TN_ATTN_V, "v", il, "bias")); + layer.o_b = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "v", il, "bias")); + layer.ln_1_b = get_tensor(new_clip->ctx, format(TN_LN_1, "v", il, "bias")); + layer.ln_2_b = get_tensor(new_clip->ctx, format(TN_LN_2, "v", il, "bias")); + layer.ff_i_b = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "v", il, "bias")); + layer.ff_o_b = get_tensor(new_clip->ctx, format(TN_FFN_UP, "v", il, "bias")); + } } - ggml_free(ctx_data); + ggml_free(meta); gguf_free(ctx); - return true; + return new_clip; } // utility function for a workaround until https://github.com/ggerganov/ggml/issues/260 is resolved // after that, remove this and use the mechanism implemented in GGML directly size_t get_mem_req_by_size(const size_t n_tensors, const int n_image_positions) { size_t mb = 1024 * 1024; + return 24 * mb; switch (n_tensors) { case 397: // base if (n_image_positions == 50) { // patch size = 32 @@ -291,6 +450,7 @@ size_t get_mem_req_by_size(const size_t n_tensors, const int n_image_positions) size_t get_scr_buf_req_by_size(const size_t n_tensors, const int n_positions) { size_t mb = 1024 * 1024; + return 96 * mb; switch (n_tensors) { case 397: if (n_positions <= 50) { @@ -548,8 +708,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { printf("%s: loading model from '%s' - please wait...", __func__, fname); } - load_gguf(fname); - exit(0); auto fin = std::ifstream(fname, std::ios::binary); if (!fin) { diff --git a/clip.h b/clip.h index dbddb07..8046b43 100644 --- a/clip.h +++ b/clip.h @@ -42,6 +42,7 @@ struct clip_tokens { }; struct clip_ctx * clip_model_load(const char * fname, const int verbosity); +struct clip_ctx * clip_model_load_gguf(const char * fname, const int verbosity); void clip_free(struct clip_ctx * ctx); diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py index ac554fe..a3183bb 100644 --- a/models/convert_hf_to_gguf.py +++ b/models/convert_hf_to_gguf.py @@ -32,7 +32,7 @@ def get_tensor_name(name: str) -> str: if "projection" in name: return name - return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "fc_up").replace("mlp.fc2", "fc_down") + return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln") def bytes_to_unicode(): @@ -195,7 +195,6 @@ def bytes_to_unicode(): fout.write_header_to_file() fout.write_kv_data_to_file() -fout.write_ti_data_to_file() fout.write_tensors_to_file() fout.close() From 186223852b547aa53fd753b8c31423b58ba6ed23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 24 Sep 2023 08:38:34 +0300 Subject: [PATCH 03/23] Check model capabilities + use new GGUF model loading function in examples --- clip.cpp | 41 +++++++++++++++++++++++++++++++---------- clip.h | 2 +- examples/main.cpp | 6 +++++- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/clip.cpp b/clip.cpp index eb7071f..45d3700 100644 --- a/clip.cpp +++ b/clip.cpp @@ -189,7 +189,7 @@ struct clip_ctx { }; // read and create ggml_context containing the tensors and their data -struct clip_ctx * clip_model_load_gguf(const char * fname, const int verbosity = 1) { +struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { struct ggml_context * meta = NULL; @@ -392,7 +392,7 @@ struct clip_ctx * clip_model_load_gguf(const char * fname, const int verbosity = vision_model.pre_ln_b = get_tensor(new_clip->ctx, format(TN_LN_PRE, "v", "bias")); vision_model.post_ln_w = get_tensor(new_clip->ctx, format(TN_LN_POST, "v", "weight")); vision_model.post_ln_b = get_tensor(new_clip->ctx, format(TN_LN_POST, "v", "bias")); - vision_model.projection = get_tensor(new_clip->ctx, TN_TEXT_PROJ); + vision_model.projection = get_tensor(new_clip->ctx, TN_VIS_PROJ); vision_model.layers.resize(hparams.n_layer); for (int il = 0; il < hparams.n_layer; ++il) { auto & layer = vision_model.layers[il]; @@ -704,7 +704,7 @@ void clip_image_batch_preprocess(const clip_ctx * ctx, const int n_threads, cons } } -struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { +struct clip_ctx * clip_model_load_legacy(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { printf("%s: loading model from '%s' - please wait...", __func__, fname); } @@ -1233,6 +1233,11 @@ void clip_free(clip_ctx * ctx) { bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_tokens * tokens, float * vec, const bool normalize) { + if (!ctx->has_text_encoder) { + printf("This GGUF file seems to have no text encoder\n"); + return false; + } + const auto & model = ctx->text_model; const auto & hparams = model.hparams; const size_t N = tokens->size; @@ -1445,6 +1450,11 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke } bool clip_image_encode(const clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec, const bool normalize) { + if (!ctx->has_vision_encoder) { + printf("This gguf file seems to have no vision encoder\n"); + return false; + } + clip_image_f32_batch imgs{}; imgs.size = 1; imgs.data = img; @@ -1453,6 +1463,12 @@ bool clip_image_encode(const clip_ctx * ctx, const int n_threads, clip_image_f32 bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs, float * vec, const bool normalize) { + + if (!ctx->has_vision_encoder) { + printf("This gguf file seems to have no vision encoder\n"); + return false; + } + const auto & model = ctx->vision_model; const auto & hparams = model.hparams; @@ -1734,11 +1750,23 @@ float clip_similarity_score(const float * vec1, const float * vec2, const int ve bool clip_compare_text_and_image(const clip_ctx * ctx, const int n_threads, const char * text, const clip_image_u8 * image, float * score) { + if (ctx->has_text_encoder && ctx->has_vision_encoder) { + printf("clip_compare_text_enc_image function can only be used with two-tower models\n"); + return false; + } + // prepare image and text vectors const int projection_dim = ctx->vision_model.hparams.projection_dim; float img_vec[projection_dim]; float txt_vec[projection_dim]; + // tokenize and encode text + auto tokens = clip_tokenize(ctx, text); + + if (!clip_text_encode(ctx, n_threads, &tokens, txt_vec, true)) { + return false; + } + // preprocess and encode image clip_image_f32 img_res; @@ -1750,13 +1778,6 @@ bool clip_compare_text_and_image(const clip_ctx * ctx, const int n_threads, cons return false; } - // tokenize and encode text - auto tokens = clip_tokenize(ctx, text); - - if (!clip_text_encode(ctx, n_threads, &tokens, txt_vec, true)) { - return false; - } - // compute similarity *score = clip_similarity_score(img_vec, txt_vec, projection_dim); diff --git a/clip.h b/clip.h index 8046b43..0071b30 100644 --- a/clip.h +++ b/clip.h @@ -41,8 +41,8 @@ struct clip_tokens { size_t size; }; +struct clip_ctx * clip_model_load_legacy(const char * fname, const int verbosity); struct clip_ctx * clip_model_load(const char * fname, const int verbosity); -struct clip_ctx * clip_model_load_gguf(const char * fname, const int verbosity); void clip_free(struct clip_ctx * ctx); diff --git a/examples/main.cpp b/examples/main.cpp index 2792673..d4f8388 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -36,7 +36,11 @@ int main(int argc, char ** argv) { const char * text = params.texts[0].c_str(); float score; - clip_compare_text_and_image(ctx, params.n_threads, text, &img0, &score); + if (!clip_compare_text_and_image(ctx, params.n_threads, text, &img0, &score)) { + printf("Unable to compare text and image\n"); + clip_free(ctx); + return 1; + } const int64_t t_main_end_us = ggml_time_us(); From 8ee5c94fedc253242584329ac1877925a7c6fb3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 24 Sep 2023 12:24:32 +0300 Subject: [PATCH 04/23] More model capability check + actually use image_mean & image_std from gguf --- clip.cpp | 65 ++++++++++++++++++++++---------- clip.h | 2 +- examples/image-search/search.cpp | 3 +- examples/main.c | 5 ++- tests/benchmark.cpp | 3 +- 5 files changed, 54 insertions(+), 24 deletions(-) diff --git a/clip.cpp b/clip.cpp index 45d3700..ba5ffd4 100644 --- a/clip.cpp +++ b/clip.cpp @@ -182,6 +182,8 @@ struct clip_ctx { struct clip_text_model text_model; struct clip_vision_model vision_model; struct clip_vocab vocab; + float image_mean[3]; + float image_std[3]; int32_t use_gelu = 0; int32_t ftype = 1; struct ggml_context * ctx; @@ -358,8 +360,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } } - new_clip->buf_compute.resize(24 * 1024 * 1024); - // vision model if (new_clip->has_vision_encoder) { // load vision model @@ -373,6 +373,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { hparams.patch_size = get_int(ctx, "clip.vision.patch_size"); hparams.projection_dim = get_int(ctx, "clip.vision.projection_dim"); + int idx_mean = get_key_idx(ctx, "clip.vision.image_mean"); + int idx_std = get_key_idx(ctx, "clip.vision.image_std"); + for (int i = 0; i < 3; ++i) { + new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean)); + new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std)); + } + if (verbosity >= 2) { printf("\n%s: vision model hparams\n", __func__); @@ -418,6 +425,8 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { ggml_free(meta); gguf_free(ctx); + new_clip->buf_compute.resize(24 * 1024 * 1024); + return new_clip; } @@ -473,7 +482,12 @@ size_t get_scr_buf_req_by_size(const size_t n_tensors, const int n_positions) { } } -struct clip_tokens clip_tokenize(const clip_ctx * ctx, const char * text) { +bool clip_tokenize(const clip_ctx * ctx, const char * text, struct clip_tokens * tokens) { + if (!ctx->has_text_encoder) { + printf("This GGUF file seems to have no text encoder\n"); + return false; + } + std::vector words; // first split the text into words @@ -506,8 +520,8 @@ struct clip_tokens clip_tokenize(const clip_ctx * ctx, const char * text) { } } - std::vector tokens; - tokens.push_back(49406); // startoftext + std::vector v_tokens; + v_tokens.push_back(49406); // startoftext for (const auto & word : words) { // feel lucky? let's try if it's a full word @@ -521,7 +535,7 @@ struct clip_tokens clip_tokenize(const clip_ctx * ctx, const char * text) { full_word += ""; auto wit = ctx->vocab.token_to_id.find(full_word); if (wit != ctx->vocab.token_to_id.end()) { - tokens.push_back(wit->second); + v_tokens.push_back(wit->second); continue; } @@ -530,7 +544,7 @@ struct clip_tokens clip_tokenize(const clip_ctx * ctx, const char * text) { auto cand = word.substr(i, j - i + 1); auto it = ctx->vocab.token_to_id.find(cand); if (it != ctx->vocab.token_to_id.end()) { // word.substr(i, j-i+1) in vocab - tokens.push_back(it->second); + v_tokens.push_back(it->second); i = j + 1; break; } else if (j == i) { // word.substr(i, 1) has no matching @@ -541,15 +555,14 @@ struct clip_tokens clip_tokenize(const clip_ctx * ctx, const char * text) { } } - tokens.push_back(49407); // endoftext + v_tokens.push_back(49407); // endoftext - clip_tokens c_tokens{}; - c_tokens.size = tokens.size(); + tokens->size = v_tokens.size(); - c_tokens.data = new int[tokens.size()]; - std::copy(tokens.begin(), tokens.end(), c_tokens.data); + tokens->data = new int[v_tokens.size()]; + std::copy(v_tokens.begin(), v_tokens.end(), tokens->data); - return c_tokens; + return true; } clip_image_u8 * make_clip_image_u8() { return new clip_image_u8(); } @@ -578,6 +591,11 @@ bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) { // normalize: x = (x - mean) / std // TODO: implement bicubic interpolation instead of linear. bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res) { + if (!ctx->has_vision_encoder) { + printf("This gguf file seems to have no vision encoder\n"); + return false; + } + const int nx = img->nx; const int ny = img->ny; @@ -594,8 +612,8 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip const int nx3 = int(nx / scale + 0.5f); const int ny3 = int(ny / scale + 0.5f); - const float m3[3] = {0.48145466f, 0.4578275f, 0.40821073f}; - const float s3[3] = {0.26862954f, 0.26130258f, 0.27577711f}; + const auto & m3 = ctx->image_mean; // {0.48145466f, 0.4578275f, 0.40821073f}; + const auto & s3 = ctx->image_std; // {0.26862954f, 0.26130258f, 0.27577711f}; for (int y = 0; y < ny3; y++) { for (int x = 0; x < nx3; x++) { @@ -1750,8 +1768,8 @@ float clip_similarity_score(const float * vec1, const float * vec2, const int ve bool clip_compare_text_and_image(const clip_ctx * ctx, const int n_threads, const char * text, const clip_image_u8 * image, float * score) { - if (ctx->has_text_encoder && ctx->has_vision_encoder) { - printf("clip_compare_text_enc_image function can only be used with two-tower models\n"); + if (!(ctx->has_text_encoder && ctx->has_vision_encoder)) { + printf("clip_compare_text_and_image function can only be used with two-tower models\n"); return false; } @@ -1761,7 +1779,10 @@ bool clip_compare_text_and_image(const clip_ctx * ctx, const int n_threads, cons float txt_vec[projection_dim]; // tokenize and encode text - auto tokens = clip_tokenize(ctx, text); + clip_tokens tokens; + if (!clip_tokenize(ctx, text, &tokens)) { + return false; + } if (!clip_text_encode(ctx, n_threads, &tokens, txt_vec, true)) { return false; @@ -1837,6 +1858,11 @@ bool softmax_with_sorting(float * arr, const int length, float * sorted_scores, bool clip_zero_shot_label_image(struct clip_ctx * ctx, const int n_threads, const struct clip_image_u8 * input_img, const char ** labels, const size_t n_labels, float * scores, int * indices) { + if (!(ctx->has_text_encoder && ctx->has_vision_encoder)) { + printf("clip_zero_shot_label_image function can only be used with two-tower models\n"); + return false; + } + // load the image clip_image_f32 img_res; @@ -1855,7 +1881,8 @@ bool clip_zero_shot_label_image(struct clip_ctx * ctx, const int n_threads, cons for (int i = 0; i < n_labels; i++) { const auto & text = labels[i]; - auto tokens = clip_tokenize(ctx, text); + clip_tokens tokens; + clip_tokenize(ctx, text, &tokens); clip_text_encode(ctx, n_threads, &tokens, txt_vec, false); similarities[i] = clip_similarity_score(img_vec, txt_vec, vec_dim); } diff --git a/clip.h b/clip.h index 0071b30..c6ee8f2 100644 --- a/clip.h +++ b/clip.h @@ -76,7 +76,7 @@ struct clip_image_f32_batch { size_t size; }; -struct clip_tokens clip_tokenize(const struct clip_ctx * ctx, const char * text); +bool clip_tokenize(const struct clip_ctx * ctx, const char * text, struct clip_tokens * tokens); struct clip_image_u8 * make_clip_image_u8(); struct clip_image_f32 * make_clip_image_f32(); diff --git a/examples/image-search/search.cpp b/examples/image-search/search.cpp index c5a125c..bfad55d 100644 --- a/examples/image-search/search.cpp +++ b/examples/image-search/search.cpp @@ -149,7 +149,8 @@ int main(int argc, char ** argv) { } } else { - auto tokens = clip_tokenize(clip_ctx, params.search_text.c_str()); + clip_tokens tokens; + clip_tokenize(clip_ctx, params.search_text.c_str(), &tokens); clip_text_encode(clip_ctx, params.n_threads, &tokens, vec.data(), true); } diff --git a/examples/main.c b/examples/main.c index 819253a..2418bea 100644 --- a/examples/main.c +++ b/examples/main.c @@ -40,11 +40,12 @@ int main() { } // Tokenize text - struct clip_tokens tokens = clip_tokenize(ctx, text); + struct clip_tokens * tokens = NULL; + clip_tokenize(ctx, text, tokens); // Encode text float txt_vec[vec_dim]; - if (!clip_text_encode(ctx, n_threads, &tokens, txt_vec, true)) { + if (!clip_text_encode(ctx, n_threads, tokens, txt_vec, true)) { fprintf(stderr, "%s: failed to encode text\n", __func__); return 1; } diff --git a/tests/benchmark.cpp b/tests/benchmark.cpp index 8e5e50d..3fa123c 100644 --- a/tests/benchmark.cpp +++ b/tests/benchmark.cpp @@ -63,7 +63,8 @@ int main(int argc, char ** argv) { const int64_t t_start_encode_texts = ggml_time_us(); for (const auto & entry : result) { - auto tokens = clip_tokenize(ctx, entry.first.c_str()); + clip_tokens tokens; + clip_tokenize(ctx, entry.first.c_str(), &tokens); if (!clip_text_encode(ctx, n_threads, &tokens, txt_vecs + label_idx * vec_dim, true)) { printf("%s: Could not encode the label at index %d: %s\n", __func__, label_idx, entry.first.c_str()); return 1; From 2fa62f33eebd4d756cb2fa9bfabfa5da86869298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 24 Sep 2023 23:46:15 +0300 Subject: [PATCH 05/23] Quantize GGUF model + support q5_0, q5_1, q8_0 --- clip.cpp | 190 +++++++++++++++++++++++++-- clip.h | 2 + models/quantize.cpp | 314 ++------------------------------------------ 3 files changed, 194 insertions(+), 312 deletions(-) diff --git a/clip.cpp b/clip.cpp index ba5ffd4..122d765 100644 --- a/clip.cpp +++ b/clip.cpp @@ -187,6 +187,7 @@ struct clip_ctx { int32_t use_gelu = 0; int32_t ftype = 1; struct ggml_context * ctx; + struct gguf_context * ctx_gguf; struct clip_buffer buf_compute; }; @@ -423,7 +424,8 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { } ggml_free(meta); - gguf_free(ctx); + + new_clip->ctx_gguf = ctx; new_clip->buf_compute.resize(24 * 1024 * 1024); @@ -1246,6 +1248,7 @@ struct clip_ctx * clip_model_load_legacy(const char * fname, const int verbosity void clip_free(clip_ctx * ctx) { ggml_free(ctx->ctx); + gguf_free(ctx->ctx_gguf); delete ctx; } @@ -1893,24 +1896,187 @@ bool clip_zero_shot_label_image(struct clip_ctx * ctx, const int n_threads, cons return true; } -bool image_normalize(const clip_image_u8 * img, clip_image_f32 * res) { - if (img->nx != 224 || img->ny != 224) { - printf("%s: long input shape: %d x %d\n", __func__, img->nx, img->ny); +bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype) { + + ggml_type type = GGML_TYPE_Q4_1; + + switch (itype) { + case 2: + type = GGML_TYPE_Q4_0; + break; + case 3: + type = GGML_TYPE_Q4_1; + break; + case 6: + type = GGML_TYPE_Q5_0; + break; + case 7: + type = GGML_TYPE_Q5_1; + break; + case 8: + type = GGML_TYPE_Q8_0; + break; + default: + fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype); return false; + }; + + auto ctx_clip = clip_model_load(fname_inp, 2); + const auto & ctx_src = ctx_clip->ctx_gguf; + const auto & ctx_data = ctx_clip->ctx; + + auto ctx_out = gguf_init_empty(); + gguf_set_kv(ctx_out, ctx_src); + gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION); + + auto fout = std::ofstream(fname_out, std::ios::binary); + + const int n_tensors = gguf_get_n_tensors(ctx_src); + + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx_src, i); + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); + gguf_add_tensor(ctx_out, cur); } - const float m3[3] = {0.48145466f, 0.4578275f, 0.40821073f}; - const float s3[3] = {0.26862954f, 0.26130258f, 0.27577711f}; + const size_t meta_size = gguf_get_meta_size(ctx_out); + for (size_t i = 0; i < meta_size; ++i) { + fout.put(0); + } - for (int y = 0; y < img->ny; y++) { - for (int x = 0; x < img->nx; x++) { - for (int c = 0; c < 3; c++) { - const int i = 3 * (y * img->nx + x) + c; - float v = (float)img->data[i]; - res->data[i] = ((v / 255.0f) - m3[c]) / s3[c]; + // regexes of tensor names to be quantized + const std::vector k_names = { + ".*weight", + }; + + std::vector read_data(512); + std::vector work(512); + std::vector conv_buf(512); + std::vector hist_all(1 << 4, 0); + size_t total_size_org = 0; + size_t total_size_new = 0; + + for (int i = 0; i < n_tensors; ++i) { + const std::string name = gguf_get_tensor_name(ctx_src, i); + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str()); + + enum ggml_type new_type; + void * new_data; + size_t new_size; + + bool quantize = false; + for (const auto & s : k_names) { + if (std::regex_match(name, std::regex(s))) { + quantize = true; + break; + } + } + + // quantize only 2D tensors + quantize &= (cur->n_dims == 2); + + if (quantize) { + new_type = type; + const size_t n_elms = ggml_nelements(cur); + float * f32_data; + + switch (cur->type) { + case GGML_TYPE_F32: + f32_data = (float *)cur->data; + break; + case GGML_TYPE_F16: + if (conv_buf.size() < n_elms) { + conv_buf.resize(n_elms); + } + for (int j = 0; j < n_elms; ++j) { + conv_buf[j] = ggml_fp16_to_fp32(((ggml_fp16_t *)cur->data)[j]); + } + f32_data = (float *)conv_buf.data(); + break; + default: + printf("Please use an input file in f32 or f16\n"); + return false; + } + + if (work.size() < n_elms * 4) { + work.resize(n_elms * 4); } + new_data = work.data(); + + std::vector hist_cur(1 << 4, 0); + + switch (new_type) { + case GGML_TYPE_Q4_0: { + new_size = ggml_quantize_q4_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q4_1: { + new_size = ggml_quantize_q4_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q5_0: { + new_size = ggml_quantize_q5_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q5_1: { + new_size = ggml_quantize_q5_1(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + case GGML_TYPE_Q8_0: { + new_size = ggml_quantize_q8_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data()); + } break; + default: { + fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, new_type); + return false; + } + } + + for (int j = 0; j < hist_cur.size(); ++j) { + hist_all[j] += hist_cur[j]; + } + } else { + new_type = cur->type; + new_data = cur->data; + new_size = ggml_nbytes(cur); } + const size_t orig_size = ggml_nbytes(cur); + total_size_org += orig_size; + total_size_new += new_size; + gguf_set_tensor_type(ctx_out, name.c_str(), new_type); + gguf_set_tensor_data(ctx_out, name.c_str(), new_data, new_size); + fout.write((const char *)new_data, new_size); + size_t pad = GGML_PAD(new_size, gguf_get_alignment(ctx_out)) - new_size; + for (int j = 0; j < pad; ++j) { + fout.put(0); + } + + printf("%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), cur->n_dims, quantize, + orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); } + + // go back to beginning of file and write the updated metadata + fout.seekp(0, std::ios::beg); + std::vector meta(meta_size); + gguf_get_meta_data(ctx_out, meta.data()); + fout.write((const char *)meta.data(), meta_size); + + fout.close(); + + clip_free(ctx_clip); + gguf_free(ctx_out); + + { + printf("%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0); + printf("%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0); + + int64_t sum_all = 0; + for (size_t i = 0; i < hist_all.size(); ++i) { + sum_all += hist_all[i]; + } + + printf("%s: hist: ", __func__); + for (size_t i = 0; i < hist_all.size(); ++i) { + printf("%5.3f ", hist_all[i] / (float)sum_all); + } + printf("\n"); + } + return true; } diff --git a/clip.h b/clip.h index c6ee8f2..5bcf477 100644 --- a/clip.h +++ b/clip.h @@ -102,6 +102,8 @@ bool softmax_with_sorting(float * arr, const int length, float * sorted_scores, bool clip_zero_shot_label_image(struct clip_ctx * ctx, const int n_threads, const struct clip_image_u8 * input_img, const char ** labels, const size_t n_labels, float * scores, int * indices); +bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype); + #ifdef __cplusplus } #endif diff --git a/models/quantize.cpp b/models/quantize.cpp index a5a2168..bc3e13c 100644 --- a/models/quantize.cpp +++ b/models/quantize.cpp @@ -11,316 +11,30 @@ #include #include -// quantize a model -bool clip_model_quantize(const std::string & fname_inp, const std::string & fname_out, int itype) { - ggml_type type = GGML_TYPE_Q4_1; - - switch (itype) { - case 2: - type = GGML_TYPE_Q4_0; - break; - case 3: - type = GGML_TYPE_Q4_1; - break; - default: - fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype); - return 1; - }; - - if (type != GGML_TYPE_Q4_0 && type != GGML_TYPE_Q4_1) { - fprintf(stderr, "%s: invalid quantization type %d\n", __func__, type); - return false; - } - - printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str()); - - auto fin = std::ifstream(fname_inp, std::ios::binary); - if (!fin) { - fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str()); - return false; - } - - auto fout = std::ofstream(fname_out, std::ios::binary); - if (!fout) { - fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str()); - return false; - } - - // verify magic - { - uint32_t magic; - fin.read((char *)&magic, sizeof(magic)); - if (magic != 0x67676d6c) { - fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str()); - return false; - } - - fout.write((char *)&magic, sizeof(magic)); - } - - clip_text_hparams t_hparams; - clip_vision_hparams v_hparams; - int32_t use_gelu; - int32_t ftype; - - // load text hparams - { - fin.read((char *)&t_hparams.n_vocab, sizeof(t_hparams.n_vocab)); - fin.read((char *)&t_hparams.num_positions, sizeof(t_hparams.num_positions)); - fin.read((char *)&t_hparams.hidden_size, sizeof(t_hparams.hidden_size)); - fin.read((char *)&t_hparams.n_intermediate, sizeof(t_hparams.n_intermediate)); - fin.read((char *)&t_hparams.projection_dim, sizeof(t_hparams.projection_dim)); - fin.read((char *)&t_hparams.n_head, sizeof(t_hparams.n_head)); - fin.read((char *)&t_hparams.n_layer, sizeof(t_hparams.n_layer)); - - printf("%s: n_vocab = %d\n", __func__, t_hparams.n_vocab); - printf("%s: num_positions = %d\n", __func__, t_hparams.num_positions); - printf("%s: t_hidden_size = %d\n", __func__, t_hparams.hidden_size); - printf("%s: t_n_intermediate = %d\n", __func__, t_hparams.n_intermediate); - printf("%s: t_n_head = %d\n", __func__, t_hparams.n_head); - printf("%s: t_n_layer = %d\n", __func__, t_hparams.n_layer); - - fout.write((char *)&t_hparams.n_vocab, sizeof(t_hparams.n_vocab)); - fout.write((char *)&t_hparams.num_positions, sizeof(t_hparams.num_positions)); - fout.write((char *)&t_hparams.hidden_size, sizeof(t_hparams.hidden_size)); - fout.write((char *)&t_hparams.n_intermediate, sizeof(t_hparams.n_intermediate)); - fout.write((char *)&t_hparams.projection_dim, sizeof(t_hparams.projection_dim)); - fout.write((char *)&t_hparams.n_head, sizeof(t_hparams.n_head)); - fout.write((char *)&t_hparams.n_layer, sizeof(t_hparams.n_layer)); - } - - // load vision hparams - { - fin.read((char *)&v_hparams.image_size, sizeof(v_hparams.image_size)); - fin.read((char *)&v_hparams.patch_size, sizeof(v_hparams.patch_size)); - fin.read((char *)&v_hparams.hidden_size, sizeof(v_hparams.hidden_size)); - fin.read((char *)&v_hparams.n_intermediate, sizeof(v_hparams.n_intermediate)); - fin.read((char *)&v_hparams.projection_dim, sizeof(v_hparams.projection_dim)); - fin.read((char *)&v_hparams.n_head, sizeof(v_hparams.n_head)); - fin.read((char *)&v_hparams.n_layer, sizeof(v_hparams.n_layer)); - fin.read((char *)&use_gelu, sizeof(use_gelu)); - fin.read((char *)&ftype, sizeof(ftype)); - - printf("%s: image_size = %d\n", __func__, v_hparams.image_size); - printf("%s: patch_size = %d\n", __func__, v_hparams.patch_size); - printf("%s: v_hidden_size = %d\n", __func__, v_hparams.hidden_size); - printf("%s: v_n_intermediate = %d\n", __func__, v_hparams.n_intermediate); - printf("%s: v_n_head = %d\n", __func__, v_hparams.n_head); - printf("%s: v_n_layer = %d\n", __func__, v_hparams.n_layer); - printf("%s: use_gelu = %d\n", __func__, use_gelu); - - fout.write((char *)&v_hparams.image_size, sizeof(v_hparams.image_size)); - fout.write((char *)&v_hparams.patch_size, sizeof(v_hparams.patch_size)); - fout.write((char *)&v_hparams.hidden_size, sizeof(v_hparams.hidden_size)); - fout.write((char *)&v_hparams.n_intermediate, sizeof(v_hparams.n_intermediate)); - fout.write((char *)&v_hparams.projection_dim, sizeof(v_hparams.projection_dim)); - fout.write((char *)&v_hparams.n_head, sizeof(v_hparams.n_head)); - fout.write((char *)&v_hparams.n_layer, sizeof(v_hparams.n_layer)); - fout.write((char *)&use_gelu, sizeof(use_gelu)); - fout.write((char *)&itype, sizeof(itype)); - } - - // load vocab - { - int32_t n_vocab = 0; - fin.read((char *)&n_vocab, sizeof(n_vocab)); - fout.write((char *)&n_vocab, sizeof(n_vocab)); - - std::string word; - for (int i = 0; i < n_vocab; i++) { - uint32_t len; - fin.read((char *)&len, sizeof(len)); - fout.write((char *)&len, sizeof(len)); - - word.resize(len); - fin.read((char *)word.data(), len); - fout.write((char *)word.data(), len); - } - } - - // load weights - { - size_t total_size_org = 0; - size_t total_size_new = 0; - - std::vector work; - - std::vector data_u8; - std::vector data_f16; - std::vector data_f32; - - std::vector hist_all(1 << 4, 0); - - while (true) { - int32_t n_dims; - int32_t length; - int32_t ftype; - - fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); - fin.read(reinterpret_cast(&length), sizeof(length)); - fin.read(reinterpret_cast(&ftype), sizeof(ftype)); - - if (fin.eof()) { - break; - } - - int32_t nelements = 1; - int32_t ne[4] = {1, 1, 1, 1}; - for (int i = 0; i < n_dims; ++i) { - fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); - nelements *= ne[i]; - } - - std::string name(length, 0); - fin.read(&name[0], length); - - { - static const char * ftype_str[] = { - "f32", - "f16", - "q4_0", - "q4_1", - }; - printf("%48s - [%5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ftype_str[ftype]); - } - - // regexes of tensor names to be quantized - const std::vector k_names = { - ".*weight", - }; - - bool quantize = false; - for (const auto & s : k_names) { - if (std::regex_match(name, std::regex(s))) { - quantize = true; - break; - } - } - - // quantize only 2D tensors - quantize &= (n_dims == 2); - - if (quantize) { - if (ftype != 0 && ftype != 1) { - fprintf(stderr, "%s: unsupported ftype %d for integer quantization\n", __func__, ftype); - return false; - } - - if (ftype == 1) { - data_f16.resize(nelements); - fin.read(reinterpret_cast(data_f16.data()), nelements * sizeof(ggml_fp16_t)); - data_f32.resize(nelements); - for (int i = 0; i < nelements; ++i) { - data_f32[i] = ggml_fp16_to_fp32(data_f16[i]); - } - } else { - data_f32.resize(nelements); - fin.read(reinterpret_cast(data_f32.data()), nelements * sizeof(float)); - } - - ftype = itype; - } else { - const int bpe = (ftype == 0) ? sizeof(float) : sizeof(uint16_t); - - data_u8.resize(nelements * bpe); - fin.read(reinterpret_cast(data_u8.data()), nelements * bpe); - } - - fout.write(reinterpret_cast(&n_dims), sizeof(n_dims)); - fout.write(reinterpret_cast(&length), sizeof(length)); - fout.write(reinterpret_cast(&ftype), sizeof(ftype)); - for (int i = 0; i < n_dims; ++i) { - fout.write(reinterpret_cast(&ne[i]), sizeof(ne[i])); - } - fout.write(&name[0], length); - - if (quantize) { - printf("quantizing .. "); - work.resize(nelements); // for quantization - - size_t cur_size = 0; - std::vector hist_cur(1 << 4, 0); - - switch (type) { - case GGML_TYPE_Q4_0: { - cur_size = ggml_quantize_q4_0(data_f32.data(), work.data(), nelements, ne[0], hist_cur.data()); - } break; - case GGML_TYPE_Q4_1: { - cur_size = ggml_quantize_q4_1(data_f32.data(), work.data(), nelements, ne[0], hist_cur.data()); - } break; - default: { - fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, type); - return false; - } - } - - fout.write(reinterpret_cast(work.data()), cur_size); - total_size_new += cur_size; - - printf("size = %8.2f MB -> %8.2f MB | hist: ", nelements * sizeof(float) / 1024.0 / 1024.0, - cur_size / 1024.0 / 1024.0); - for (size_t i = 0; i < hist_cur.size(); ++i) { - hist_all[i] += hist_cur[i]; - } - - for (size_t i = 0; i < hist_cur.size(); ++i) { - printf("%5.3f ", hist_cur[i] / (float)nelements); - } - printf("\n"); - } else { - printf("size = %8.3f MB\n", data_u8.size() / 1024.0 / 1024.0); - fout.write(reinterpret_cast(data_u8.data()), data_u8.size()); - total_size_new += data_u8.size(); - } - - total_size_org += nelements * sizeof(float) / 2.0; - } - - printf("%s: model size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0); - printf("%s: quant size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0); - - { - int64_t sum_all = 0; - for (size_t i = 0; i < hist_all.size(); ++i) { - sum_all += hist_all[i]; - } - - printf("%s: hist: ", __func__); - for (size_t i = 0; i < hist_all.size(); ++i) { - printf("%5.3f ", hist_all[i] / (float)sum_all); - } - printf("\n"); - } - } - - fin.close(); - fout.close(); - - return true; +void print_usage(int argc, char ** argv) { + + fprintf(stderr, "usage: %s /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized.gguf type\n", argv[0]); + fprintf(stderr, " type = 2 - q4_0\n"); + fprintf(stderr, " type = 3 - q4_1\n"); + fprintf(stderr, " type = 6 - q5_0\n"); + fprintf(stderr, " type = 7 - q5_1\n"); + fprintf(stderr, " type = 8 - q8_0\n"); } -// usage: -// ./bin/quantize models/ggml-model-f16.bin models/ggml-model-q4_1.bin type -// int main(int argc, char ** argv) { if (argc != 4) { - fprintf(stderr, "usage: %s /path/to/ggml-model-f16.bin /path/to/ggml-model-quantized.bin type\n", argv[0]); - fprintf(stderr, " type = 2 - q4_0\n"); - fprintf(stderr, " type = 3 - q4_1\n"); + print_usage(argc, argv); return 1; } - // needed to initialize f16 tables - { - struct ggml_init_params params = {0, NULL, false}; - struct ggml_context * ctx = ggml_init(params); - ggml_free(ctx); - } - const std::string fname_inp = argv[1]; const std::string fname_out = argv[2]; const int itype = atoi(argv[3]); + if (itype != 2 && itype != 3 && itype != 6 && itype != 7 && itype != 8) { + print_usage(argc, argv); + return 1; + } const int64_t t_main_start_us = ggml_time_us(); @@ -330,7 +44,7 @@ int main(int argc, char ** argv) { { const int64_t t_start_us = ggml_time_us(); - if (!clip_model_quantize(fname_inp, fname_out, itype)) { + if (!clip_model_quantize(fname_inp.c_str(), fname_out.c_str(), itype)) { fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str()); return 1; } From 89612961bf5420d44b50abcb007779bb26056b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 25 Sep 2023 00:45:48 +0300 Subject: [PATCH 06/23] rm legacy model loading function --- clip.cpp | 522 -------------------------------------------- clip.h | 1 - models/quantize.cpp | 9 - 3 files changed, 532 deletions(-) diff --git a/clip.cpp b/clip.cpp index 122d765..94fdba7 100644 --- a/clip.cpp +++ b/clip.cpp @@ -724,528 +724,6 @@ void clip_image_batch_preprocess(const clip_ctx * ctx, const int n_threads, cons } } -struct clip_ctx * clip_model_load_legacy(const char * fname, const int verbosity = 1) { - if (verbosity >= 1) { - printf("%s: loading model from '%s' - please wait...", __func__, fname); - } - - auto fin = std::ifstream(fname, std::ios::binary); - if (!fin) { - fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname); - return nullptr; - } - - // verify magic - { - uint32_t magic; - fin.read((char *)&magic, sizeof(magic)); - if (magic != 0x67676d6c) { - fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname); - return nullptr; - } - } - - clip_ctx * new_clip = new clip_ctx; - clip_text_model & text_model = new_clip->text_model; - clip_vision_model & vision_model = new_clip->vision_model; - clip_vocab & vocab = new_clip->vocab; - - // load hparams for text - { - auto & hparams = text_model.hparams; - - fin.read((char *)&hparams.n_vocab, sizeof(hparams.n_vocab)); - fin.read((char *)&hparams.num_positions, sizeof(hparams.num_positions)); - fin.read((char *)&hparams.hidden_size, sizeof(hparams.hidden_size)); - fin.read((char *)&hparams.n_intermediate, sizeof(hparams.n_intermediate)); - fin.read((char *)&hparams.projection_dim, sizeof(hparams.projection_dim)); - fin.read((char *)&hparams.n_head, sizeof(hparams.n_head)); - fin.read((char *)&hparams.n_layer, sizeof(hparams.n_layer)); - - if (verbosity >= 2) { - printf("\n%s: text model hparams\n", __func__); - printf("n_vocab %d\n", hparams.n_vocab); - printf("num_positions %d\n", hparams.num_positions); - printf("t_hidden_size %d\n", hparams.hidden_size); - printf("t_n_intermediate %d\n", hparams.n_intermediate); - printf("t_projection_dim %d\n", hparams.projection_dim); - printf("t_n_head %d\n", hparams.n_head); - printf("t_n_layer %d\n", hparams.n_layer); - } - } - - // load hparams for vision - { - auto & hparams = vision_model.hparams; - - fin.read((char *)&hparams.image_size, sizeof(hparams.image_size)); - fin.read((char *)&hparams.patch_size, sizeof(hparams.patch_size)); - fin.read((char *)&hparams.hidden_size, sizeof(hparams.hidden_size)); - fin.read((char *)&hparams.n_intermediate, sizeof(hparams.n_intermediate)); - fin.read((char *)&hparams.projection_dim, sizeof(hparams.projection_dim)); - fin.read((char *)&hparams.n_head, sizeof(hparams.n_head)); - fin.read((char *)&hparams.n_layer, sizeof(hparams.n_layer)); - - fin.read((char *)&new_clip->use_gelu, sizeof(new_clip->use_gelu)); - fin.read((char *)&new_clip->ftype, sizeof(new_clip->ftype)); - - if (verbosity >= 2) { - printf("\n%s: vision model hparams\n", __func__); - printf("image_size %d\n", hparams.image_size); - printf("patch_size %d\n", hparams.patch_size); - printf("v_hidden_size %d\n", hparams.hidden_size); - printf("v_n_intermediate %d\n", hparams.n_intermediate); - printf("v_projection_dim %d\n", hparams.projection_dim); - printf("v_n_head %d\n", hparams.n_head); - printf("v_n_layer %d\n", hparams.n_layer); - - printf("\nuse_gelu %d\n", new_clip->use_gelu); - printf("ftype %d\n\n", new_clip->ftype); - } - } - - // load vocab - { - int32_t n_vocab = 0; - fin.read((char *)&n_vocab, sizeof(n_vocab)); - - if (n_vocab != new_clip->text_model.hparams.n_vocab) { - fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", __func__, fname, n_vocab, - new_clip->text_model.hparams.n_vocab); - return nullptr; - } - - std::string word; - std::vector buf(128); - - for (int i = 0; i < n_vocab; i++) { - uint32_t len; - fin.read((char *)&len, sizeof(len)); - - buf.resize(len); - fin.read((char *)buf.data(), len); - word.assign(buf.data(), len); - - new_clip->vocab.token_to_id[word] = i; - new_clip->vocab.id_to_token[i] = word; - } - } - - // for the big tensors, we have the option to store the data in 16-bit floats or quantized - // in order to save memory and also to speed up the computation - ggml_type wtype = GGML_TYPE_COUNT; - switch (new_clip->ftype) { - case 0: - wtype = GGML_TYPE_F32; - break; - case 1: - wtype = GGML_TYPE_F16; - break; - case 2: - wtype = GGML_TYPE_Q4_0; - break; - case 3: - wtype = GGML_TYPE_Q4_1; - break; - default: { - fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", __func__, fname, new_clip->ftype); - clip_free(new_clip); - return nullptr; - } - } - - auto & ctx = new_clip->ctx; - size_t model_mem_req = 0; - - { - // calculate memory requirement for text_model - const auto & hparams = text_model.hparams; - - const int n_vocab = hparams.n_vocab; - const int num_positions = hparams.num_positions; - const int hidden_size = hparams.hidden_size; - const int n_layer = hparams.n_layer; - const int n_intermediate = hparams.n_intermediate; - const int projection_dim = hparams.projection_dim; - - // Calculate size requirements - - model_mem_req += hidden_size * n_vocab * ggml_type_sizef(wtype); // token_embeddings - model_mem_req += hidden_size * num_positions * ggml_type_sizef(wtype); // position_embeddings - - model_mem_req += 4 * n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // ln_1_* and ln_2_* - - model_mem_req += 4 * n_layer * (hidden_size * hidden_size * ggml_type_sizef(wtype)); // kqvo weights - model_mem_req += 4 * n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // kqvo bias - model_mem_req += 2 * n_layer * (hidden_size * n_intermediate * ggml_type_sizef(wtype)); // ff_*_w - model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b - model_mem_req += n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b - - model_mem_req += 2 * hidden_size * ggml_type_sizef(GGML_TYPE_F32); // post_ln_* - model_mem_req += 2 * hidden_size * projection_dim * ggml_type_sizef(wtype); // projection - - model_mem_req += (5 + 16 * n_layer) * 256; // object overhead - } - - { - // calculate memory requirement for vision_model - const auto & hparams = vision_model.hparams; - - const int image_size = hparams.image_size; - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size / patch_size) * (image_size / patch_size)) + 1; - const int hidden_size = hparams.hidden_size; - const int n_layer = hparams.n_layer; - const int n_intermediate = hparams.n_intermediate; - const int projection_dim = hparams.projection_dim; - - // Calculate size requirements - - model_mem_req += hidden_size * ggml_type_sizef(GGML_TYPE_F32); // class_embedding - model_mem_req += hidden_size * 3 * patch_size * patch_size * ggml_type_sizef(GGML_TYPE_F16); // patch_embeddings - model_mem_req += hidden_size * num_patches * ggml_type_sizef(wtype); // position_embeddings - - model_mem_req += 2 * hidden_size * ggml_type_sizef(GGML_TYPE_F32); // pre_ln_* - - model_mem_req += 4 * n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // ln_* - - model_mem_req += 4 * n_layer * (hidden_size * hidden_size * ggml_type_sizef(wtype)); // kqvo weights - model_mem_req += 4 * n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // kqvo bias - - model_mem_req += 2 * n_layer * (hidden_size * n_intermediate * ggml_type_sizef(wtype)); // ff_*_w - model_mem_req += n_layer * (n_intermediate * ggml_type_sizef(GGML_TYPE_F32)); // ff_i_b - model_mem_req += n_layer * (hidden_size * ggml_type_sizef(GGML_TYPE_F32)); // ff_o_b - - model_mem_req += 2 * hidden_size * ggml_type_sizef(GGML_TYPE_F32); // post_ln_* - model_mem_req += 2 * hidden_size * projection_dim * ggml_type_sizef(wtype); // projection - - model_mem_req += (5 + 16 * n_layer) * 256; // object overhead - } - - if (verbosity >= 2) { - printf("%s: ggml ctx size = %6.2f MB\n", __func__, model_mem_req / (1024.0 * 1024.0)); - } - - // create the ggml context - { - struct ggml_init_params params = { - .mem_size = model_mem_req, - .mem_buffer = NULL, - .no_alloc = false, - }; - - new_clip->ctx = ggml_init(params); - if (!new_clip->ctx) { - fprintf(stderr, "%s: ggml_init() failed\n", __func__); - clip_free(new_clip); - return nullptr; - } - } - - // prepare memory for the text_model weights - { - const auto & hparams = text_model.hparams; - - const int n_vocab = hparams.n_vocab; - const int num_positions = hparams.num_positions; - const int hidden_size = hparams.hidden_size; - const int n_layer = hparams.n_layer; - const int n_intermediate = hparams.n_intermediate; - const int projection_dim = hparams.projection_dim; - - text_model.layers.resize(n_layer); - - text_model.token_embeddings = ggml_new_tensor_2d(ctx, wtype, hidden_size, n_vocab); - text_model.position_embeddings = ggml_new_tensor_2d(ctx, wtype, hidden_size, num_positions); - - // map by name - text_model.tensors["text_model.embeddings.token_embedding.weight"] = text_model.token_embeddings; - text_model.tensors["text_model.embeddings.position_embedding.weight"] = text_model.position_embeddings; - - for (int i = 0; i < n_layer; ++i) { - auto & layer = text_model.layers[i]; - - layer.ln_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - layer.q_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.k_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.v_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.o_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - layer.ff_i_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, n_intermediate); - layer.ff_i_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_intermediate); - - layer.ff_o_w = ggml_new_tensor_2d(ctx, wtype, n_intermediate, hidden_size); - layer.ff_o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - // map by name - - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.k_proj.weight"] = layer.k_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.k_proj.bias"] = layer.k_b; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.v_proj.weight"] = layer.v_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.v_proj.bias"] = layer.v_b; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.q_proj.weight"] = layer.q_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.q_proj.bias"] = layer.q_b; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.out_proj.weight"] = layer.o_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".self_attn.out_proj.bias"] = layer.o_b; - - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".layer_norm1.weight"] = layer.ln_1_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".layer_norm1.bias"] = layer.ln_1_b; - - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".mlp.fc1.weight"] = layer.ff_i_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".mlp.fc1.bias"] = layer.ff_i_b; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".mlp.fc2.weight"] = layer.ff_o_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".mlp.fc2.bias"] = layer.ff_o_b; - - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".layer_norm2.weight"] = layer.ln_2_w; - text_model.tensors["text_model.encoder.layers." + std::to_string(i) + ".layer_norm2.bias"] = layer.ln_2_b; - } - - text_model.post_ln_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - text_model.post_ln_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - text_model.projection = ggml_new_tensor_2d(ctx, wtype, hidden_size, projection_dim); - - // map by name - text_model.tensors["text_model.final_layer_norm.weight"] = text_model.post_ln_w; - text_model.tensors["text_model.final_layer_norm.bias"] = text_model.post_ln_b; - text_model.tensors["text_projection.weight"] = text_model.projection; - } - - // prepare memory for the vision_model weights - { - const auto & hparams = vision_model.hparams; - - const int image_size = hparams.image_size; - const int patch_size = hparams.patch_size; - const int num_patches = ((image_size / patch_size) * (image_size / patch_size)) + 1; - const int hidden_size = hparams.hidden_size; - const int n_layer = hparams.n_layer; - const int n_intermediate = hparams.n_intermediate; - const int projection_dim = hparams.projection_dim; - - vision_model.layers.resize(n_layer); - - vision_model.class_embedding = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - vision_model.patch_embeddings = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, patch_size, patch_size, 3, hidden_size); - vision_model.position_embeddings = ggml_new_tensor_2d(ctx, wtype, hidden_size, num_patches); - - vision_model.pre_ln_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - vision_model.pre_ln_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - // map by name - vision_model.tensors["vision_model.embeddings.class_embedding"] = vision_model.class_embedding; - vision_model.tensors["vision_model.embeddings.patch_embedding.weight"] = vision_model.patch_embeddings; - vision_model.tensors["vision_model.embeddings.position_embedding.weight"] = vision_model.position_embeddings; - - vision_model.tensors["vision_model.pre_layrnorm.weight"] = vision_model.pre_ln_w; - vision_model.tensors["vision_model.pre_layrnorm.bias"] = vision_model.pre_ln_b; - - for (int i = 0; i < n_layer; ++i) { - auto & layer = vision_model.layers[i]; - - layer.ln_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - layer.q_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.k_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.v_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - layer.o_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, hidden_size); - layer.o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - layer.ff_i_w = ggml_new_tensor_2d(ctx, wtype, hidden_size, n_intermediate); - layer.ff_i_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_intermediate); - - layer.ff_o_w = ggml_new_tensor_2d(ctx, wtype, n_intermediate, hidden_size); - layer.ff_o_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - - // map by name - - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.k_proj.weight"] = layer.k_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.k_proj.bias"] = layer.k_b; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.v_proj.weight"] = layer.v_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.v_proj.bias"] = layer.v_b; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.q_proj.weight"] = layer.q_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.q_proj.bias"] = layer.q_b; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.out_proj.weight"] = layer.o_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".self_attn.out_proj.bias"] = layer.o_b; - - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".layer_norm1.weight"] = layer.ln_1_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".layer_norm1.bias"] = layer.ln_1_b; - - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".mlp.fc1.weight"] = layer.ff_i_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".mlp.fc1.bias"] = layer.ff_i_b; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".mlp.fc2.weight"] = layer.ff_o_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".mlp.fc2.bias"] = layer.ff_o_b; - - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".layer_norm2.weight"] = layer.ln_2_w; - vision_model.tensors["vision_model.encoder.layers." + std::to_string(i) + ".layer_norm2.bias"] = layer.ln_2_b; - } - - vision_model.post_ln_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - vision_model.post_ln_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden_size); - vision_model.projection = ggml_new_tensor_2d(ctx, wtype, hidden_size, projection_dim); - - // map by name - vision_model.tensors["vision_model.post_layernorm.weight"] = vision_model.post_ln_w; - vision_model.tensors["vision_model.post_layernorm.bias"] = vision_model.post_ln_b; - vision_model.tensors["visual_projection.weight"] = vision_model.projection; - } - - // load weights - { - int n_tensors = 0; - size_t total_size = 0; - - while (true) { - int32_t n_dims; - int32_t length; - int32_t ftype; - - fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); - fin.read(reinterpret_cast(&length), sizeof(length)); - fin.read(reinterpret_cast(&ftype), sizeof(ftype)); - - if (fin.eof()) { - break; - } - - int64_t nelements = 1; - int64_t ne[4] = {1, 1, 1, 1}; - for (int i = 0; i < n_dims; ++i) { - int32_t ne_cur; - fin.read(reinterpret_cast(&ne_cur), sizeof(ne_cur)); - ne[i] = ne_cur; - nelements *= ne[i]; - } - - std::string name(length, 0); - fin.read(&name[0], length); - - struct ggml_tensor * tensor; - if (text_model.tensors.find(name.data()) != text_model.tensors.end()) { - tensor = text_model.tensors[name.data()]; - } else if (vision_model.tensors.find(name.data()) != vision_model.tensors.end()) { - tensor = vision_model.tensors[name.data()]; - } else { - fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data()); - clip_free(new_clip); - return nullptr; - } - - if (ggml_nelements(tensor) != nelements) { - fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.data()); - clip_free(new_clip); - return nullptr; - } - - if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { - fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%ld, %ld], expected [%ld, %ld]\n", - __func__, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]); - clip_free(new_clip); - return nullptr; - } - - if (0) { - static const char * ftype_str[] = { - "f32", - "f16", - "q4_0", - "q4_1", - }; - printf("%24s - [%5ld, %5ld], type = %6s, %6.2f MB, %9zu bytes\n", name.data(), ne[0], ne[1], ftype_str[ftype], - ggml_nbytes(tensor) / 1024.0 / 1024.0, ggml_nbytes(tensor)); - } - - size_t bpe = 0; - - switch (ftype) { - case 0: - bpe = ggml_type_size(GGML_TYPE_F32); - break; - case 1: - bpe = ggml_type_size(GGML_TYPE_F16); - break; - case 2: - bpe = ggml_type_size(GGML_TYPE_Q4_0); - assert(ne[0] % 64 == 0); - break; - case 3: - bpe = ggml_type_size(GGML_TYPE_Q4_1); - assert(ne[0] % 64 == 0); - break; - default: { - fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype); - clip_free(new_clip); - return nullptr; - } - }; - - if ((nelements * bpe) / ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { - fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", __func__, name.data(), - ggml_nbytes(tensor), nelements * bpe); - clip_free(new_clip); - return nullptr; - } - - fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); - -#ifdef CLIP_DEBUG_TENSORS - printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", - ggml_nbytes(tensor) / 1024.0 / 1024.0); -#endif - - total_size += ggml_nbytes(tensor); - if (verbosity >= 1) { - if (++n_tensors % 8 == 0) { - printf("."); - fflush(stdout); - } - } - } - - if (verbosity >= 1) { - printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size / 1024.0 / 1024.0, n_tensors); - } - } - - fin.close(); - - // Calculate space requirements for setting up context buffers later - { - // TODO: We currently get the size of memory requirement from the pre-computed information - // based on the model variant, indicated by the number of tensors. - // Rewrite this logic when GGML implements a mechanism to predict the required memory. - const size_t n_tensors = new_clip->text_model.tensors.size() + new_clip->vision_model.tensors.size(); - const int n_image_positions = (vision_model.hparams.image_size / vision_model.hparams.patch_size) * - (vision_model.hparams.image_size / vision_model.hparams.patch_size) + - 1; - size_t mem_req = get_mem_req_by_size(n_tensors, n_image_positions); - new_clip->buf_compute.resize(mem_req); - - if (verbosity >= 2) { - printf("%s: %zu MB of compute buffer allocated\n", __func__, mem_req / 1024 / 1024); - } - } - - if (verbosity >= 1) { - printf("%s: model loaded\n\n", __func__); - } - - return new_clip; -} - void clip_free(clip_ctx * ctx) { ggml_free(ctx->ctx); gguf_free(ctx->ctx_gguf); diff --git a/clip.h b/clip.h index 5bcf477..9653800 100644 --- a/clip.h +++ b/clip.h @@ -41,7 +41,6 @@ struct clip_tokens { size_t size; }; -struct clip_ctx * clip_model_load_legacy(const char * fname, const int verbosity); struct clip_ctx * clip_model_load(const char * fname, const int verbosity); void clip_free(struct clip_ctx * ctx); diff --git a/models/quantize.cpp b/models/quantize.cpp index bc3e13c..83f150c 100644 --- a/models/quantize.cpp +++ b/models/quantize.cpp @@ -1,15 +1,6 @@ #include "clip.h" #include "ggml/ggml.h" - -#include -#include -#include -#include -#include -#include -#include #include -#include void print_usage(int argc, char ** argv) { From 57f00e2f5703bb9ba435c9d341872f9d4ab530b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 25 Sep 2023 10:49:12 +0300 Subject: [PATCH 07/23] some fixes and more friendly logging --- clip.cpp | 87 +++++++++++++++++++++++++----------- models/convert_hf_to_gguf.py | 39 +++++++++------- 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/clip.cpp b/clip.cpp index 94fdba7..943d763 100644 --- a/clip.cpp +++ b/clip.cpp @@ -17,7 +17,7 @@ #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" -#define CLIP_DEBUG +// #define CLIP_DEBUG static std::string format(const char * fmt, ...) { va_list ap; @@ -61,10 +61,10 @@ int get_key_idx(const gguf_context * ctx, char * key) { return i; } -const int get_int(const gguf_context * ctx, char * key) { +const uint32_t get_int(const gguf_context * ctx, char * key) { const int i = get_key_idx(ctx, key); - return gguf_get_val_i32(ctx, i); + return gguf_get_val_u32(ctx, i); } struct ggml_tensor * get_tensor(struct ggml_context * ctx, std::string name) { @@ -77,6 +77,34 @@ struct ggml_tensor * get_tensor(struct ggml_context * ctx, std::string name) { return cur; } +std::string get_ftype(int ftype) { + switch (ftype) { + case 0: + return "f32"; + break; + case 1: + return "f16"; + break; + case 2: + return "q4_0"; + break; + case 3: + return "q4_1"; + break; + case 6: + return "q5_0"; + break; + case 7: + return "q5_1"; + break; + case 8: + return "q8_0"; + break; + default: + throw std::runtime_error(format("Unrecognized file type: %d\n", ftype)); + } +} + // // Vocab utils // @@ -137,8 +165,6 @@ struct clip_text_model { struct ggml_tensor * post_ln_b; struct ggml_tensor * projection; - - std::map tensors; }; struct clip_vision_model { @@ -158,8 +184,6 @@ struct clip_vision_model { struct ggml_tensor * post_ln_b; struct ggml_tensor * projection; - - std::map tensors; }; // Replacement for std::vector that doesn't require zero-initialization. @@ -184,7 +208,7 @@ struct clip_ctx { struct clip_vocab vocab; float image_mean[3]; float image_std[3]; - int32_t use_gelu = 0; + bool use_gelu = false; int32_t ftype = 1; struct ggml_context * ctx; struct gguf_context * ctx_gguf; @@ -203,21 +227,29 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { struct gguf_context * ctx = gguf_init_from_file(fname, params); - printf("%s: version: %d\n", __func__, gguf_get_version(ctx)); - printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); - printf("%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); + if (verbosity >= 1) { + const int n_tensors = gguf_get_n_tensors(ctx); + const int n_kv = gguf_get_n_kv(ctx); + const int ftype = get_int(ctx, "general.file_type"); + const std::string ftype_str = get_ftype(ftype); + printf("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx)); + printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); + printf("%s: n_tensors: %d\n", __func__, n_tensors); + printf("%s: n_kv: %d\n", __func__, n_kv); + printf("%s: ftype: %s\n", __func__, ftype_str.c_str()); + printf("\n"); + } // kv - if (verbosity >= 2) { + if (verbosity >= 3) { const int n_kv = gguf_get_n_kv(ctx); - printf("%s: n_kv: %d\n", __func__, n_kv); - for (int i = 0; i < n_kv; ++i) { const char * key = gguf_get_key(ctx, i); printf("%s: kv[%d]: key = %s\n", __func__, i, key); } + printf("\n"); } // data @@ -234,21 +266,16 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { size_t tensor_size = ggml_nbytes(cur); size_t padded_size = ggml_nbytes_pad(cur); ctx_size += padded_size; - if (verbosity >= 2) { + if (verbosity >= 3) { printf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, padded_size=%zu, offset=%zu\n", __func__, i, cur->n_dims, cur->name, tensor_size, padded_size, offset); } } - - if (verbosity >= 1) { - printf("%s: ctx_meta size: %zu\n", __func__, ggml_get_mem_size(meta)); - printf("%s: ctx_data size: %zu (%f MB)\n", __func__, ctx_size, (ctx_size / (1024.0f * 1024.0f))); - } } clip_ctx * new_clip = new clip_ctx; - // model capabilities + // model size and capabilities { int idx = get_key_idx(ctx, "clip.has_text_encoder"); new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx); @@ -256,8 +283,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { idx = get_key_idx(ctx, "clip.has_vision_encoder"); new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx); - printf("%s: has_text_encoder: %d\n", __func__, new_clip->has_text_encoder); - printf("%s: has_vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); + idx = get_key_idx(ctx, "clip.use_gelu"); + new_clip->use_gelu = gguf_get_val_bool(ctx, idx); + + if (verbosity >= 1) { + printf("%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder); + printf("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); + printf("%s: ctx_data size: %f MB\n", __func__, (ctx_size / 1024.0 / 1024.0)); + printf("%s: ctx_meta size: %f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0); + } } // load tensors @@ -296,6 +330,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { clip_free(new_clip); return nullptr; } + fin.read(reinterpret_cast(cur->data), ggml_nbytes(t)); } @@ -761,7 +796,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke struct ggml_context * ctx0 = ggml_init(params); struct ggml_cgraph gf = {}; - static size_t scr0_size = get_scr_buf_req_by_size(ctx->text_model.tensors.size() + ctx->vision_model.tensors.size(), N); + static size_t scr0_size = get_scr_buf_req_by_size(gguf_get_n_tensors(ctx->ctx_gguf), N); static void * scr0 = malloc(scr0_size); struct ggml_tensor * input_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); @@ -994,8 +1029,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl struct ggml_context * ctx0 = ggml_init(params); struct ggml_cgraph gf = {}; - static size_t scr0_size = - get_scr_buf_req_by_size(ctx->text_model.tensors.size() + ctx->vision_model.tensors.size(), num_positions); + static size_t scr0_size = get_scr_buf_req_by_size(gguf_get_n_tensors(ctx->ctx_gguf), num_positions); static void * scr0 = malloc(scr0_size); struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size, image_size, 3, batch_size); @@ -1406,6 +1440,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i auto ctx_out = gguf_init_empty(); gguf_set_kv(ctx_out, ctx_src); gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION); + gguf_set_val_u32(ctx_out, "general.file_type", itype); auto fout = std::ofstream(fname_out, std::ios::binary); diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py index a3183bb..4a06a9c 100644 --- a/models/convert_hf_to_gguf.py +++ b/models/convert_hf_to_gguf.py @@ -122,35 +122,44 @@ def bytes_to_unicode(): fout.add_bool("clip.has_text_encoder", has_text_encoder) fout.add_bool("clip.has_vision_encoder", has_vision_encoder) +fout.add_file_type(ftype) +model_name = config["name_or_path"] if "name_or_path" in config else dir_model.split(os.path.sep)[-1] +fout.add_name(model_name) +if args.text_only: + fout.add_description("text-only CLIP model") +elif args.vision_only: + fout.add_description("vision-only CLIP model") +else: + fout.add_description("two-tower CLIP model") if has_text_encoder: # text_model hparams - fout.add_int32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"]) - fout.add_int32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"]) - fout.add_int32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"]) - fout.add_int32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"])) - fout.add_int32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"]) + fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"]) + fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"]) + fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"]) + fout.add_uint32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"])) + fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"]) fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"]) - fout.add_int32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"]) + fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"]) fout.add_token_list(tokens) if has_vision_encoder: # vision_model hparams - fout.add_int32("clip.vision.image_size", v_hparams["image_size"]) - fout.add_int32("clip.vision.patch_size", v_hparams["patch_size"]) - fout.add_int32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"]) - fout.add_int32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"]) - fout.add_int32("clip.vision.projection_dim", v_hparams.get("projection_dim", config["projection_dim"])) - fout.add_int32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"]) + fout.add_uint32("clip.vision.image_size", v_hparams["image_size"]) + fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"]) + fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"]) + fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"]) + fout.add_uint32("clip.vision.projection_dim", v_hparams.get("projection_dim", config["projection_dim"])) + fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"]) fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"]) - fout.add_int32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"]) + fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"]) image_mean = processor.image_processor.image_mean if args.image_mean is None else args.image_mean image_std = processor.image_processor.image_std if args.image_std is None else args.image_std fout.add_array("clip.vision.image_mean", image_mean) fout.add_array("clip.vision.image_std", image_std) -use_gelu = int(v_hparams["hidden_act"] == "gelu") +use_gelu = v_hparams["hidden_act"] == "gelu" fout.add_bool("clip.use_gelu", use_gelu) @@ -198,4 +207,4 @@ def bytes_to_unicode(): fout.write_tensors_to_file() fout.close() -print("Done. Output file: " + fname_out) \ No newline at end of file +print("Done. Output file: " + fname_out) From 1a344e8b7a5181e82ef9c5f0a51f453bf82024de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 25 Sep 2023 14:20:18 +0300 Subject: [PATCH 08/23] add example for extracting vectors as .npy file --- examples/CMakeLists.txt | 4 ++ examples/common-clip.cpp | 80 +++++++++++++++++++++++++++++++++++++--- examples/common-clip.h | 10 +++-- examples/extract.cpp | 80 ++++++++++++++++++++++++++++++++++++++++ examples/main.cpp | 6 +-- examples/zsl.cpp | 4 +- 6 files changed, 169 insertions(+), 15 deletions(-) create mode 100644 examples/extract.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6a8c394..c47b60d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -18,3 +18,7 @@ target_link_libraries(zsl PRIVATE clip common-clip ggml) add_executable(main_c main.c) target_compile_features(main_c PUBLIC c_std_11) target_link_libraries(main_c PRIVATE clip ggml) + +add_executable(extract extract.cpp) +target_compile_features(extract PUBLIC cxx_std_11) +target_link_libraries(extract PRIVATE clip common-clip ggml) diff --git a/examples/common-clip.cpp b/examples/common-clip.cpp index b520d9b..ffe8ddf 100644 --- a/examples/common-clip.cpp +++ b/examples/common-clip.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include #include @@ -138,7 +140,7 @@ bool is_image_file_extension(const std::string & path) { return false; } -bool app_params_parse(int argc, char ** argv, app_params & params) { +bool app_params_parse(int argc, char ** argv, app_params & params, const int min_text_arg, const int min_image_arg) { for (int i = 0; i < argc; i++) { std::string arg = std::string(argv[i]); if (arg == "-m" || arg == "--model") { @@ -152,7 +154,7 @@ bool app_params_parse(int argc, char ** argv, app_params & params) { } else if (arg == "-v" || arg == "--verbose") { params.verbose = std::stoi(argv[++i]); } else if (arg == "-h" || arg == "--help") { - print_help(argc, argv, params); + print_help(argc, argv, params, min_text_arg, min_image_arg); exit(0); } else { if (i != 0) { @@ -161,17 +163,27 @@ bool app_params_parse(int argc, char ** argv, app_params & params) { } } } - return params.image_paths.size() >= 1 && params.texts.size() >= 1; + return params.image_paths.size() >= min_image_arg && params.texts.size() >= min_text_arg; } -void print_help(int argc, char ** argv, app_params & params) { +void print_help(int argc, char ** argv, app_params & params, const int min_text_arg, const int min_image_arg) { printf("Usage: %s [options]\n", argv[0]); printf("\nOptions:"); printf(" -h, --help: Show this message and exit\n"); printf(" -m , --model : path to model. Default: %s\n", params.model.c_str()); printf(" -t N, --threads N: Number of threads to use for inference. Default: %d\n", params.n_threads); - printf(" --text : Text to encode. At least one text should be specified\n"); - printf(" --image : Path to an image file. At least one image path should be specified\n"); + printf(" --text : Text to encode"); + if (min_text_arg >= 1) { + printf(". At least %d text should be specified\n", min_text_arg); + } else { + printf("\n"); + } + printf(" --image : Path to an image file"); + if (min_image_arg >= 1) { + printf(". At least one image path should be specified\n"); + } else { + printf("\n"); + } printf(" -v , --verbose : Control the level of verbosity. 0 = minimum, 2 = maximum. Default: %d\n", params.verbose); } @@ -193,6 +205,62 @@ void write_floats_to_file(float * array, int size, char * filename) { fclose(file); } +// Define a structure to represent the numpy header +typedef struct { + uint8_t major_version; + uint8_t minor_version; + uint16_t header_len; + char * header[128]; +} NumpyHeader; + +// Function to write a numpy .npy file +int writeNpyFile(const char * filename, const float * data, const int * shape, int ndims) { + if (ndims != 2) { + printf("Only 2-dimensional arrays are supported for now\n"); + return 1; + } + + FILE * file = fopen(filename, "wb"); + if (!file) { + perror("Error opening file"); + return 1; + } + + // Create the numpy header + NumpyHeader header; + header.major_version = 1; + header.minor_version = 0; + + // Calculate the length of the header data + snprintf((char *)header.header, sizeof(header.header), "{'descr': ' & images) { struct clip_image_u8_batch batch; diff --git a/examples/common-clip.h b/examples/common-clip.h index 2aea563..3fea56a 100644 --- a/examples/common-clip.h +++ b/examples/common-clip.h @@ -27,14 +27,16 @@ struct app_params { int verbose; app_params() - : n_threads(std::min(4, static_cast(std::thread::hardware_concurrency()))), model("models/ggml-model-f16.bin"), - verbose(1) { + : n_threads(std::min(4, static_cast(std::thread::hardware_concurrency()))), + model("models/ggml-model-f16.gguf"), verbose(1) { // Initialize other fields if needed } }; -bool app_params_parse(int argc, char ** argv, app_params & params); -void print_help(int argc, char ** argv, app_params & params); +bool app_params_parse(int argc, char ** argv, app_params & params, const int min_text_arg, const int min_image_arg); +void print_help(int argc, char ** argv, app_params & params, const int min_text_arg, const int min_image_arg); + +int writeNpyFile(const char * filename, const float * data, const int * shape, int ndims); // utils for debugging void write_floats_to_file(float * array, int size, char * filename); diff --git a/examples/extract.cpp b/examples/extract.cpp new file mode 100644 index 0000000..1032428 --- /dev/null +++ b/examples/extract.cpp @@ -0,0 +1,80 @@ +// extract vectors of texts or image +// TODO: support multiple text or multiple images +// optionally from a file + +#include "clip.h" +#include "common-clip.h" + +int main(int argc, char ** argv) { + ggml_time_init(); + const int64_t t_main_start_us = ggml_time_us(); + + app_params params; + if (!app_params_parse(argc, argv, params, 0, 0)) { + print_help(argc, argv, params, 0, 0); + return 1; + } + + if (params.image_paths.size() >= 1 && params.texts.size() >= 1) { + printf("Provide either --text or --image argument\n"); + return 1; + } + + auto ctx = clip_model_load(params.model.c_str(), params.verbose); + if (!ctx) { + printf("%s: Unable to load model from %s", __func__, params.model.c_str()); + return 1; + } + + const int64_t t_image_load_us = ggml_time_us(); + if (params.image_paths.size() >= 1) { + // load the image + const char * img_path = params.image_paths[0].c_str(); + clip_image_u8 img_input; + if (!clip_image_load_from_file(img_path, &img_input)) { + fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, img_path); + clip_free(ctx); + return 1; + } + + clip_image_f32 img_res; + if (!clip_image_preprocess(ctx, &img_input, &img_res)) { + printf("Unable to preprocess image\n"); + clip_free(ctx); + return 1; + } + + const int vec_dim = clip_get_vision_hparams(ctx)->projection_dim; + int shape[2] = {1, vec_dim}; + float vec[vec_dim]; + clip_image_encode(ctx, params.n_threads, &img_res, vec, false); + writeNpyFile("./img_vec.npy", vec, shape, 2); + + printf("Wrote to ./img_vec.npy\n"); + } else { + const char * text = params.texts[0].c_str(); + clip_tokens tokens; + if (!clip_tokenize(ctx, text, &tokens)) { + printf("Unable to tokenize text\n"); + clip_free(ctx); + return 1; + } + + const int vec_dim = clip_get_text_hparams(ctx)->projection_dim; + int shape[2] = {1, vec_dim}; + float vec[vec_dim]; + + if (!clip_text_encode(ctx, params.n_threads, &tokens, vec, false)) { + printf("Unable to encode text\n"); + clip_free(ctx); + return 1; + } + + writeNpyFile("./text_vec.npy", vec, shape, 2); + printf("Wrote to ./text_vec.npy\n"); + } + + clip_free(ctx); + + return 0; +} diff --git a/examples/main.cpp b/examples/main.cpp index d4f8388..26b483f 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -8,8 +8,8 @@ int main(int argc, char ** argv) { const int64_t t_main_start_us = ggml_time_us(); app_params params; - if (!app_params_parse(argc, argv, params)) { - print_help(argc, argv, params); + if (!app_params_parse(argc, argv, params, 1, 1)) { + print_help(argc, argv, params, 1, 1); return 1; } @@ -55,7 +55,7 @@ int main(int argc, char ** argv) { } // Usage of the individual functions that make up clip_compare_text_and_image is demonstrated in the - // zero-shot labelling and image search examples. + // `extract` example. clip_free(ctx); diff --git a/examples/zsl.cpp b/examples/zsl.cpp index 2972b18..b13f832 100644 --- a/examples/zsl.cpp +++ b/examples/zsl.cpp @@ -5,8 +5,8 @@ int main(int argc, char ** argv) { app_params params; - if (!app_params_parse(argc, argv, params)) { - print_help(argc, argv, params); + if (!app_params_parse(argc, argv, params, 2, 1)) { + print_help(argc, argv, params, 2, 1); return 1; } From fa26cb3a83d5da3d82ce03fd052df0e3cd8790d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 25 Sep 2023 23:21:57 +0300 Subject: [PATCH 09/23] define key constants and update handle text-only in mem_rq calculation --- clip.cpp | 141 ++++++++++++++++++++++++++----------------- examples/extract.cpp | 3 - 2 files changed, 87 insertions(+), 57 deletions(-) diff --git a/clip.cpp b/clip.cpp index 943d763..0a50b1c 100644 --- a/clip.cpp +++ b/clip.cpp @@ -34,6 +34,24 @@ static std::string format(const char * fmt, ...) { return std::string(buf.data(), buf.size()); } +// key constants +#define KEY_FTYPE "general.file_type" +#define KEY_HAS_TEXT_ENC "clip.has_text_encoder" +#define KEY_HAS_VIS_ENC "clip.has_vision_encoder" +#define KEY_USE_GELU "clip.use_gelu" +#define KEY_N_EMBD "clip.%s.embedding_length" +#define KEY_N_FF "clip.%s.feed_forward_length" +#define KEY_N_BLOCK "clip.%s.block_count" +#define KEY_N_HEAD "clip.%s.attention.head_count" +#define KEY_PROJ_DIM "clip.%s.projection_dim" +#define KEY_TOKENS "tokenizer.ggml.tokens" +#define KEY_N_POSITIONS "clip.text.context_length" +#define KEY_IMAGE_SIZE "clip.vision.image_size" +#define KEY_PATCH_SIZE "clip.vision.patch_size" +#define KEY_IMAGE_MEAN "clip.vision.image_mean" +#define KEY_IMAGE_STD "clip.vision.image_std" + +// tensor name constants #define TN_TOKEN_EMBD "%s.token_embd.weight" #define TN_POS_EMBD "%s.position_embd.weight" #define TN_CLASS_EMBD "v.class_embd" @@ -51,7 +69,7 @@ static std::string format(const char * fmt, ...) { #define TN_TEXT_PROJ "text_projection.weight" #define TN_VIS_PROJ "visual_projection.weight" -int get_key_idx(const gguf_context * ctx, char * key) { +int get_key_idx(const gguf_context * ctx, const char * key) { int i = gguf_find_key(ctx, key); if (i == -1) { // fprintf(stderr, "key %s not found in file\n", key); @@ -61,8 +79,8 @@ int get_key_idx(const gguf_context * ctx, char * key) { return i; } -const uint32_t get_int(const gguf_context * ctx, char * key) { - const int i = get_key_idx(ctx, key); +const uint32_t get_int(const gguf_context * ctx, std::string key) { + const int i = get_key_idx(ctx, key.c_str()); return gguf_get_val_u32(ctx, i); } @@ -215,6 +233,45 @@ struct clip_ctx { struct clip_buffer buf_compute; }; +// utility function for a workaround until https://github.com/ggerganov/ggml/issues/260 is resolved +// after that, remove this and use the mechanism implemented in GGML directly +size_t get_mem_req_by_size(struct clip_ctx * ctx) { + size_t mb = 1024 * 1024; + const int n_tensors = gguf_get_n_tensors(ctx->ctx_gguf); + const auto & vision_hparams = clip_get_vision_hparams(ctx); + const int n_positions = + ctx->has_vision_encoder ? vision_hparams->image_size * vision_hparams->image_size / vision_hparams->patch_size + 1 : 77; + switch (n_tensors) { + case 397: // base, two-tower + case 200: // base, vision-only + if (n_positions == 50) { // patch size = 32 + return 12 * mb; + } else { // patch size = 16 + return 24 * mb; + } + case 197: // base, text-only + return 12 * mb; + case 589: // large, two-tower + case 296: // large, vision-only + if (n_positions == 257) { // input image size = 224 + return 24 * mb; + } else { // input image size = 336 + return 60 * mb; + } + case 193: // large, text-only + return 24 * mb; + case 909: // huge, two-tower + case 456: // huge, vision-only + return 232 * mb; + case 453: // huge, text-only + return 120 * mb; + default: + fprintf(stderr, "%s: Unrecognized number of tensors: %d. Check if you pass the correct model file\n", __func__, + n_tensors); + exit(1); + } +} + // read and create ggml_context containing the tensors and their data struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { @@ -230,7 +287,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { const int n_tensors = gguf_get_n_tensors(ctx); const int n_kv = gguf_get_n_kv(ctx); - const int ftype = get_int(ctx, "general.file_type"); + const int ftype = get_int(ctx, KEY_FTYPE); const std::string ftype_str = get_ftype(ftype); printf("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx)); printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); @@ -277,13 +334,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // model size and capabilities { - int idx = get_key_idx(ctx, "clip.has_text_encoder"); + int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC); new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx); - idx = get_key_idx(ctx, "clip.has_vision_encoder"); + idx = get_key_idx(ctx, KEY_HAS_VIS_ENC); new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx); - idx = get_key_idx(ctx, "clip.use_gelu"); + idx = get_key_idx(ctx, KEY_USE_GELU); new_clip->use_gelu = gguf_get_val_bool(ctx, idx); if (verbosity >= 1) { @@ -342,14 +399,14 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // load text model auto & text_model = new_clip->text_model; auto & hparams = text_model.hparams; - hparams.hidden_size = get_int(ctx, "clip.text.embedding_length"); - hparams.n_head = get_int(ctx, "clip.text.attention.head_count"); - hparams.n_intermediate = get_int(ctx, "clip.text.feed_forward_length"); - hparams.n_layer = get_int(ctx, "clip.text.block_count"); - hparams.num_positions = get_int(ctx, "clip.text.context_length"); - hparams.projection_dim = get_int(ctx, "clip.text.projection_dim"); - - const int idx_tokens = get_key_idx(ctx, "tokenizer.ggml.tokens"); + hparams.hidden_size = get_int(ctx, format(KEY_N_EMBD, "text")); + hparams.n_head = get_int(ctx, format(KEY_N_HEAD, "text")); + hparams.n_intermediate = get_int(ctx, format(KEY_N_FF, "text")); + hparams.n_layer = get_int(ctx, format(KEY_N_BLOCK, "text")); + hparams.num_positions = get_int(ctx, KEY_N_POSITIONS); + hparams.projection_dim = get_int(ctx, format(KEY_PROJ_DIM, "text")); + + const int idx_tokens = get_key_idx(ctx, KEY_TOKENS); hparams.n_vocab = gguf_get_arr_n(ctx, idx_tokens); auto & vocab = new_clip->vocab; for (int id = 0; id < hparams.n_vocab; ++id) { @@ -401,23 +458,22 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // load vision model auto & vision_model = new_clip->vision_model; auto & hparams = vision_model.hparams; - hparams.hidden_size = get_int(ctx, "clip.vision.embedding_length"); - hparams.n_head = get_int(ctx, "clip.vision.attention.head_count"); - hparams.n_intermediate = get_int(ctx, "clip.vision.feed_forward_length"); - hparams.n_layer = get_int(ctx, "clip.vision.block_count"); - hparams.image_size = get_int(ctx, "clip.vision.image_size"); - hparams.patch_size = get_int(ctx, "clip.vision.patch_size"); - hparams.projection_dim = get_int(ctx, "clip.vision.projection_dim"); - - int idx_mean = get_key_idx(ctx, "clip.vision.image_mean"); - int idx_std = get_key_idx(ctx, "clip.vision.image_std"); + hparams.hidden_size = get_int(ctx, format(KEY_N_EMBD, "vision")); + hparams.n_head = get_int(ctx, format(KEY_N_HEAD, "vision")); + hparams.n_intermediate = get_int(ctx, format(KEY_N_FF, "vision")); + hparams.n_layer = get_int(ctx, format(KEY_N_BLOCK, "vision")); + hparams.image_size = get_int(ctx, KEY_IMAGE_SIZE); + hparams.patch_size = get_int(ctx, KEY_PATCH_SIZE); + hparams.projection_dim = get_int(ctx, format(KEY_PROJ_DIM, "vision")); + + int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN); + int idx_std = get_key_idx(ctx, KEY_IMAGE_STD); for (int i = 0; i < 3; ++i) { new_clip->image_mean[i] = *((float *)gguf_get_arr_data(ctx, idx_mean)); new_clip->image_std[i] = *((float *)gguf_get_arr_data(ctx, idx_std)); } if (verbosity >= 2) { - printf("\n%s: vision model hparams\n", __func__); printf("image_size %d\n", hparams.image_size); printf("patch_size %d\n", hparams.patch_size); @@ -462,38 +518,15 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { new_clip->ctx_gguf = ctx; - new_clip->buf_compute.resize(24 * 1024 * 1024); + const size_t mem_req = get_mem_req_by_size(new_clip); + new_clip->buf_compute.resize(mem_req); + if (verbosity >= 1) { + printf("\n%s: %zu MB of memory allocated\n", __func__, mem_req / 1024 / 1024); + } return new_clip; } -// utility function for a workaround until https://github.com/ggerganov/ggml/issues/260 is resolved -// after that, remove this and use the mechanism implemented in GGML directly -size_t get_mem_req_by_size(const size_t n_tensors, const int n_image_positions) { - size_t mb = 1024 * 1024; - return 24 * mb; - switch (n_tensors) { - case 397: // base - if (n_image_positions == 50) { // patch size = 32 - return 12 * mb; - } else { // patch size = 16 - return 24 * mb; - } - case 589: // large - if (n_image_positions == 257) { // input image size = 224 - return 24 * mb; - } else { // input image size = 336 - return 60 * mb; - } - case 909: // huge - return 232 * mb; - default: - fprintf(stderr, "%s: Unrecognized number of tensors: %zu. Check if you pass the correct model file\n", __func__, - n_tensors); - exit(1); - } -} - size_t get_scr_buf_req_by_size(const size_t n_tensors, const int n_positions) { size_t mb = 1024 * 1024; return 96 * mb; diff --git a/examples/extract.cpp b/examples/extract.cpp index 1032428..212ce35 100644 --- a/examples/extract.cpp +++ b/examples/extract.cpp @@ -6,9 +6,6 @@ #include "common-clip.h" int main(int argc, char ** argv) { - ggml_time_init(); - const int64_t t_main_start_us = ggml_time_us(); - app_params params; if (!app_params_parse(argc, argv, params, 0, 0)) { print_help(argc, argv, params, 0, 0); From bbe522bd6508ea3cf778554a4e921b7983b3b380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 01:23:03 +0300 Subject: [PATCH 10/23] print model name and description + some fixes --- clip.cpp | 90 +++++++++++++------- models/convert_hf_to_ggml.py | 158 ----------------------------------- models/convert_hf_to_gguf.py | 2 +- 3 files changed, 63 insertions(+), 187 deletions(-) delete mode 100644 models/convert_hf_to_ggml.py diff --git a/clip.cpp b/clip.cpp index 0a50b1c..8c3edb8 100644 --- a/clip.cpp +++ b/clip.cpp @@ -34,8 +34,13 @@ static std::string format(const char * fmt, ...) { return std::string(buf.data(), buf.size()); } +// // key constants +// + #define KEY_FTYPE "general.file_type" +#define KEY_NAME "general.name" +#define KEY_DESCRIPTION "general.description" #define KEY_HAS_TEXT_ENC "clip.has_text_encoder" #define KEY_HAS_VIS_ENC "clip.has_vision_encoder" #define KEY_USE_GELU "clip.use_gelu" @@ -51,7 +56,10 @@ static std::string format(const char * fmt, ...) { #define KEY_IMAGE_MEAN "clip.vision.image_mean" #define KEY_IMAGE_STD "clip.vision.image_std" +// // tensor name constants +// + #define TN_TOKEN_EMBD "%s.token_embd.weight" #define TN_POS_EMBD "%s.position_embd.weight" #define TN_CLASS_EMBD "v.class_embd" @@ -69,10 +77,14 @@ static std::string format(const char * fmt, ...) { #define TN_TEXT_PROJ "text_projection.weight" #define TN_VIS_PROJ "visual_projection.weight" +// +// utilities to get data from a gguf file +// + int get_key_idx(const gguf_context * ctx, const char * key) { int i = gguf_find_key(ctx, key); if (i == -1) { - // fprintf(stderr, "key %s not found in file\n", key); + fprintf(stderr, "key %s not found in file\n", key); throw std::runtime_error(format("Missing required key: %s", key)); } @@ -233,6 +245,10 @@ struct clip_ctx { struct clip_buffer buf_compute; }; +// +// memory allocation and management +// + // utility function for a workaround until https://github.com/ggerganov/ggml/issues/260 is resolved // after that, remove this and use the mechanism implemented in GGML directly size_t get_mem_req_by_size(struct clip_ctx * ctx) { @@ -272,6 +288,42 @@ size_t get_mem_req_by_size(struct clip_ctx * ctx) { } } +size_t get_scr_buf_req_by_size(struct clip_ctx * ctx) { + size_t mb = 1024 * 1024; + + const int n_tensors = gguf_get_n_tensors(ctx->ctx_gguf); + const auto & vision_hparams = clip_get_vision_hparams(ctx); + const int n_positions = + ctx->has_vision_encoder ? vision_hparams->image_size * vision_hparams->image_size / vision_hparams->patch_size + 1 : 77; + + switch (n_tensors) { + case 397: + case 200: + if (n_positions <= 50) { + return 32 * mb; + } else { + return 96 * mb; + } + case 197: + return 32 * mb; + case 589: + case 392: + if (n_positions <= 257) { + return 96 * mb; + } else { + return 192 * mb; + } + case 909: + case 456: + case 453: + return 144 * mb; + default: + fprintf(stderr, "%s: Unrecognized number of tensors: %d. Check if you pass the correct model file\n", __func__, + n_tensors); + exit(1); + } +} + // read and create ggml_context containing the tensors and their data struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { @@ -289,6 +341,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { const int n_kv = gguf_get_n_kv(ctx); const int ftype = get_int(ctx, KEY_FTYPE); const std::string ftype_str = get_ftype(ftype); + const int idx_name = get_key_idx(ctx, KEY_NAME); + const std::string name = gguf_get_val_str(ctx, idx_name); + const int idx_desc = get_key_idx(ctx, KEY_DESCRIPTION); + const std::string description = gguf_get_val_str(ctx, idx_desc); + + printf("%s: model name: %s\n", __func__, name.c_str()); + printf("%s: description: %s\n", __func__, description.c_str()); printf("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx)); printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); printf("%s: n_tensors: %d\n", __func__, n_tensors); @@ -527,31 +586,6 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { return new_clip; } -size_t get_scr_buf_req_by_size(const size_t n_tensors, const int n_positions) { - size_t mb = 1024 * 1024; - return 96 * mb; - switch (n_tensors) { - case 397: - if (n_positions <= 50) { - return 32 * mb; - } else { - return 96 * mb; - } - case 589: - if (n_positions <= 257) { - return 96 * mb; - } else { - return 192 * mb; - } - case 909: - return 144 * mb; - default: - fprintf(stderr, "%s: Unrecognized number of tensors: %zu. Check if you pass the correct model file\n", __func__, - n_tensors); - exit(1); - } -} - bool clip_tokenize(const clip_ctx * ctx, const char * text, struct clip_tokens * tokens) { if (!ctx->has_text_encoder) { printf("This GGUF file seems to have no text encoder\n"); @@ -829,7 +863,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke struct ggml_context * ctx0 = ggml_init(params); struct ggml_cgraph gf = {}; - static size_t scr0_size = get_scr_buf_req_by_size(gguf_get_n_tensors(ctx->ctx_gguf), N); + static size_t scr0_size = get_scr_buf_req_by_size((struct clip_ctx *)ctx); static void * scr0 = malloc(scr0_size); struct ggml_tensor * input_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); @@ -1062,7 +1096,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl struct ggml_context * ctx0 = ggml_init(params); struct ggml_cgraph gf = {}; - static size_t scr0_size = get_scr_buf_req_by_size(gguf_get_n_tensors(ctx->ctx_gguf), num_positions); + static size_t scr0_size = get_scr_buf_req_by_size((struct clip_ctx *)ctx); static void * scr0 = malloc(scr0_size); struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size, image_size, 3, batch_size); diff --git a/models/convert_hf_to_ggml.py b/models/convert_hf_to_ggml.py deleted file mode 100644 index c4c6473..0000000 --- a/models/convert_hf_to_ggml.py +++ /dev/null @@ -1,158 +0,0 @@ -import sys -import struct -import json -import torch -import numpy as np - -from transformers import CLIPModel - - -def bytes_to_unicode(): - """ - Returns list of utf-8 byte and a corresponding list of unicode strings. - The reversible bpe codes work on unicode strings. - This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. - When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. - This is a signficant percentage of your normal, say, 32K bpe vocab. - To avoid that, we want lookup tables between utf-8 bytes and unicode strings. - And avoids mapping to whitespace/control characters the bpe code barfs on. - """ - bs = ( - list(range(ord("!"), ord("~") + 1)) - + list(range(ord("¡"), ord("¬") + 1)) - + list(range(ord("®"), ord("ÿ") + 1)) - ) - cs = bs[:] - n = 0 - for b in range(2**8): - if b not in bs: - bs.append(b) - cs.append(2**8 + n) - n += 1 - cs = [chr(n) for n in cs] - return dict(zip(bs, cs)) - - -if len(sys.argv) < 3: - print("Usage: convert-h5-to-ggml.py dir-model ftype\n") - print(" ftype == 0 -> float32") - print(" ftype == 1 -> float16") - sys.exit(1) - -# output in the same directory as the model -dir_model = sys.argv[1] - - -with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f: - encoder = json.load(f) - -with open(dir_model + "/config.json", "r", encoding="utf-8") as f: - config = json.load(f) - v_hparams = config["vision_config"] - t_hparams = config["text_config"] - -# possible data types -# ftype == 0 -> float32 -# ftype == 1 -> float16 -# -# map from ftype to string -ftype_str = ["f32", "f16"] - -ftype = 1 -if len(sys.argv) > 2: - ftype = int(sys.argv[2]) - if ftype < 0 or ftype > 1: - print("Invalid ftype: " + str(ftype)) - sys.exit(1) - fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin" - - -model = CLIPModel.from_pretrained(dir_model) -# print (model) - -list_vars = model.state_dict() -# print (list_vars) - -fout = open(fname_out, "wb") - -fout.write(struct.pack("i", 0x67676D6C)) # magic: ggml in hex - -# text_model hparams -fout.write(struct.pack("i", t_hparams["vocab_size"])) -fout.write(struct.pack("i", t_hparams["max_position_embeddings"])) -fout.write(struct.pack("i", t_hparams["hidden_size"])) -fout.write(struct.pack("i", t_hparams["intermediate_size"])) -fout.write(struct.pack("i", t_hparams.get("projection_dim", config["projection_dim"]))) -fout.write(struct.pack("i", t_hparams["num_attention_heads"])) -fout.write(struct.pack("i", t_hparams["num_hidden_layers"])) - -# vision_model hparams -fout.write(struct.pack("i", v_hparams["image_size"])) -fout.write(struct.pack("i", v_hparams["patch_size"])) -fout.write(struct.pack("i", v_hparams["hidden_size"])) -fout.write(struct.pack("i", v_hparams["intermediate_size"])) -fout.write(struct.pack("i", v_hparams.get("projection_dim", config["projection_dim"]))) -fout.write(struct.pack("i", v_hparams["num_attention_heads"])) -fout.write(struct.pack("i", v_hparams["num_hidden_layers"])) - -use_gelu = int(v_hparams["hidden_act"] == "gelu") -fout.write(struct.pack("i", use_gelu)) -fout.write(struct.pack("i", ftype)) - - -byte_encoder = bytes_to_unicode() -byte_decoder = {v: k for k, v in byte_encoder.items()} -fout.write(struct.pack("i", len(encoder))) - -for key in encoder: - text = bytearray([byte_decoder[c] for c in key]) - fout.write(struct.pack("i", len(text))) - fout.write(text) - - -for name in list_vars.keys(): - if name in [ - "logit_scale", - "text_model.embeddings.position_ids", - "vision_model.embeddings.position_ids", - ]: - # we don't need this - print(f"skipping parameter: {name}") - continue - - data = list_vars[name].squeeze().numpy() - print("Processing variable: " + name + " with shape: ", data.shape) - - n_dims = len(data.shape) - - # ftype == 0 -> float32, ftype == 1 -> float16 - ftype_cur = 0 - if ftype != 0: - if name[-7:] == ".weight" and n_dims >= 2: - print(" Converting to float16") - data = data.astype(np.float16) - ftype_cur = 1 - else: - print(" Converting to float32") - data = data.astype(np.float32) - ftype_cur = 0 - else: - if data.dtype != np.float32: - print(" Converting to float32") - data = data.astype(np.float32) - ftype_cur = 0 - - # header - str = name.encode("utf-8") - fout.write(struct.pack("iii", n_dims, len(str), ftype_cur)) - for i in range(n_dims): - fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) - fout.write(str) - - # data - data.tofile(fout) - -fout.close() - -print("Done. Output file: " + fname_out) -print("") diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py index 4a06a9c..0f83b5c 100644 --- a/models/convert_hf_to_gguf.py +++ b/models/convert_hf_to_gguf.py @@ -123,7 +123,7 @@ def bytes_to_unicode(): fout.add_bool("clip.has_text_encoder", has_text_encoder) fout.add_bool("clip.has_vision_encoder", has_vision_encoder) fout.add_file_type(ftype) -model_name = config["name_or_path"] if "name_or_path" in config else dir_model.split(os.path.sep)[-1] +model_name = config["_name_or_path"] if "_name_or_path" in config else dir_model.split(os.path.sep)[-1] fout.add_name(model_name) if args.text_only: fout.add_description("text-only CLIP model") From 4bfc1b935f34613fa61b65ecb3ce600cab0c5582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 02:03:12 +0300 Subject: [PATCH 11/23] python: use new clip_tokenize signature --- examples/python_bindings/clip_cpp/clip.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/python_bindings/clip_cpp/clip.py b/examples/python_bindings/clip_cpp/clip.py index d6df3d9..e58f428 100644 --- a/examples/python_bindings/clip_cpp/clip.py +++ b/examples/python_bindings/clip_cpp/clip.py @@ -114,8 +114,8 @@ class ClipContext(ctypes.Structure): clip_get_vision_hparams.restype = ctypes.POINTER(ClipVisionHparams) clip_tokenize = clip_lib.clip_tokenize -clip_tokenize.argtypes = [ctypes.POINTER(ClipContext), ctypes.c_char_p] -clip_tokenize.restype = ClipTokens +clip_tokenize.argtypes = [ctypes.POINTER(ClipContext), ctypes.c_char_p, ctypes.POINTER(ClipTokens)] +clip_tokenize.restype = ctypes.c_bool clip_image_load_from_file = clip_lib.clip_image_load_from_file clip_image_load_from_file.argtypes = [ctypes.c_char_p, ctypes.POINTER(ClipImageU8)] @@ -310,8 +310,11 @@ def text_config(self) -> Dict[str, Any]: return _struct_to_dict(clip_get_text_hparams(self.ctx).contents) def tokenize(self, text: str) -> List[int]: - tokens = clip_tokenize(self.ctx, text.encode("utf8")) - return [tokens.data[i] for i in range(tokens.size)] + tokens = ClipTokens() + if clip_tokenize(self.ctx, text.encode("utf8"), ctypes.pointer(tokens)): + return [tokens.data[i] for i in range(tokens.size)] + else: + raise RuntimeError("unable to tokenize text") def encode_text( self, From 78a22c4b97ead86fdd47a66ba59173448745e9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 11:06:03 +0300 Subject: [PATCH 12/23] mem alignment fixes for larger models --- clip.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/clip.cpp b/clip.cpp index 8c3edb8..80057cd 100644 --- a/clip.cpp +++ b/clip.cpp @@ -265,21 +265,19 @@ size_t get_mem_req_by_size(struct clip_ctx * ctx) { } else { // patch size = 16 return 24 * mb; } - case 197: // base, text-only + case 197: // base or large, text-only return 12 * mb; case 589: // large, two-tower - case 296: // large, vision-only + case 392: // large, vision-only if (n_positions == 257) { // input image size = 224 return 24 * mb; } else { // input image size = 336 return 60 * mb; } - case 193: // large, text-only - return 24 * mb; case 909: // huge, two-tower - case 456: // huge, vision-only + case 520: // huge, vision-only return 232 * mb; - case 453: // huge, text-only + case 389: // huge, text-only return 120 * mb; default: fprintf(stderr, "%s: Unrecognized number of tensors: %d. Check if you pass the correct model file\n", __func__, @@ -314,9 +312,10 @@ size_t get_scr_buf_req_by_size(struct clip_ctx * ctx) { return 192 * mb; } case 909: - case 456: - case 453: + case 520: return 144 * mb; + case 389: + return 60 * mb; default: fprintf(stderr, "%s: Unrecognized number of tensors: %d. Check if you pass the correct model file\n", __func__, n_tensors); @@ -405,8 +404,8 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { printf("%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder); printf("%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder); - printf("%s: ctx_data size: %f MB\n", __func__, (ctx_size / 1024.0 / 1024.0)); - printf("%s: ctx_meta size: %f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0); + printf("%s: model size: %.2f MB\n", __func__, (ctx_size / 1024.0 / 1024.0)); + printf("%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0); } } From af41146fa41b310e178e10355d18549d439dc159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 11:38:40 +0300 Subject: [PATCH 13/23] use eps from gguf --- clip.cpp | 55 +++++++++++++++++++++++++++++++++---------------------- clip.h | 8 ++------ 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/clip.cpp b/clip.cpp index 80057cd..30afd09 100644 --- a/clip.cpp +++ b/clip.cpp @@ -48,6 +48,7 @@ static std::string format(const char * fmt, ...) { #define KEY_N_FF "clip.%s.feed_forward_length" #define KEY_N_BLOCK "clip.%s.block_count" #define KEY_N_HEAD "clip.%s.attention.head_count" +#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon" #define KEY_PROJ_DIM "clip.%s.projection_dim" #define KEY_TOKENS "tokenizer.ggml.tokens" #define KEY_N_POSITIONS "clip.text.context_length" @@ -91,12 +92,18 @@ int get_key_idx(const gguf_context * ctx, const char * key) { return i; } -const uint32_t get_int(const gguf_context * ctx, std::string key) { +const uint32_t get_u32(const gguf_context * ctx, std::string key) { const int i = get_key_idx(ctx, key.c_str()); return gguf_get_val_u32(ctx, i); } +const float get_f32(const gguf_context * ctx, std::string key) { + const int i = get_key_idx(ctx, key.c_str()); + + return gguf_get_val_f32(ctx, i); +} + struct ggml_tensor * get_tensor(struct ggml_context * ctx, std::string name) { struct ggml_tensor * cur = ggml_get_tensor(ctx, name.c_str()); if (!cur) { @@ -338,7 +345,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { if (verbosity >= 1) { const int n_tensors = gguf_get_n_tensors(ctx); const int n_kv = gguf_get_n_kv(ctx); - const int ftype = get_int(ctx, KEY_FTYPE); + const int ftype = get_u32(ctx, KEY_FTYPE); const std::string ftype_str = get_ftype(ftype); const int idx_name = get_key_idx(ctx, KEY_NAME); const std::string name = gguf_get_val_str(ctx, idx_name); @@ -457,12 +464,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // load text model auto & text_model = new_clip->text_model; auto & hparams = text_model.hparams; - hparams.hidden_size = get_int(ctx, format(KEY_N_EMBD, "text")); - hparams.n_head = get_int(ctx, format(KEY_N_HEAD, "text")); - hparams.n_intermediate = get_int(ctx, format(KEY_N_FF, "text")); - hparams.n_layer = get_int(ctx, format(KEY_N_BLOCK, "text")); - hparams.num_positions = get_int(ctx, KEY_N_POSITIONS); - hparams.projection_dim = get_int(ctx, format(KEY_PROJ_DIM, "text")); + hparams.hidden_size = get_u32(ctx, format(KEY_N_EMBD, "text")); + hparams.n_head = get_u32(ctx, format(KEY_N_HEAD, "text")); + hparams.n_intermediate = get_u32(ctx, format(KEY_N_FF, "text")); + hparams.n_layer = get_u32(ctx, format(KEY_N_BLOCK, "text")); + hparams.num_positions = get_u32(ctx, KEY_N_POSITIONS); + hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "text")); + hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "text")); const int idx_tokens = get_key_idx(ctx, KEY_TOKENS); hparams.n_vocab = gguf_get_arr_n(ctx, idx_tokens); @@ -516,13 +524,14 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { // load vision model auto & vision_model = new_clip->vision_model; auto & hparams = vision_model.hparams; - hparams.hidden_size = get_int(ctx, format(KEY_N_EMBD, "vision")); - hparams.n_head = get_int(ctx, format(KEY_N_HEAD, "vision")); - hparams.n_intermediate = get_int(ctx, format(KEY_N_FF, "vision")); - hparams.n_layer = get_int(ctx, format(KEY_N_BLOCK, "vision")); - hparams.image_size = get_int(ctx, KEY_IMAGE_SIZE); - hparams.patch_size = get_int(ctx, KEY_PATCH_SIZE); - hparams.projection_dim = get_int(ctx, format(KEY_PROJ_DIM, "vision")); + hparams.hidden_size = get_u32(ctx, format(KEY_N_EMBD, "vision")); + hparams.n_head = get_u32(ctx, format(KEY_N_HEAD, "vision")); + hparams.n_intermediate = get_u32(ctx, format(KEY_N_FF, "vision")); + hparams.n_layer = get_u32(ctx, format(KEY_N_BLOCK, "vision")); + hparams.image_size = get_u32(ctx, KEY_IMAGE_SIZE); + hparams.patch_size = get_u32(ctx, KEY_PATCH_SIZE); + hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "vision")); + hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision")); int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN); int idx_std = get_key_idx(ctx, KEY_IMAGE_STD); @@ -850,6 +859,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke const int n_layer = hparams.n_layer; const int n_intermediate = hparams.n_intermediate; const int projection_dim = hparams.projection_dim; + const float eps = hparams.eps; auto & buf_compute = ctx->buf_compute; @@ -885,7 +895,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke // layernorm1 { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, eps); cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_1_w, cur), cur), ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); @@ -935,7 +945,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke // layernorm2 { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, eps); cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_2_w, cur), cur), ggml_repeat(ctx0, model.layers[il].ln_2_b, cur)); @@ -961,7 +971,7 @@ bool clip_text_encode(const clip_ctx * ctx, const int n_threads, const clip_toke // final -layer_norm { - embeddings = ggml_norm(ctx0, embeddings, 1e-5f); + embeddings = ggml_norm(ctx0, embeddings, eps); embeddings = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.post_ln_w, embeddings), embeddings), ggml_repeat(ctx0, model.post_ln_b, embeddings)); @@ -1082,6 +1092,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl const int n_layer = hparams.n_layer; const int n_intermediate = hparams.n_intermediate; const int projection_dim = hparams.projection_dim; + const float eps = hparams.eps; int batch_size = imgs->size; auto & buf_compute = ctx->buf_compute; @@ -1148,7 +1159,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl // pre-layernorm { - embeddings = ggml_norm(ctx0, embeddings, 1e-5f); + embeddings = ggml_norm(ctx0, embeddings, eps); embeddings = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.pre_ln_w, embeddings), embeddings), ggml_repeat(ctx0, model.pre_ln_b, embeddings)); @@ -1164,7 +1175,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl // layernorm1 { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, eps); cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_1_w, cur), cur), ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); @@ -1214,7 +1225,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl // layernorm2 { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, eps); cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_2_w, cur), cur), ggml_repeat(ctx0, model.layers[il].ln_2_b, cur)); @@ -1247,7 +1258,7 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl // post-layernorm { - embeddings = ggml_norm(ctx0, embeddings, 1e-4f); + embeddings = ggml_norm(ctx0, embeddings, eps); embeddings = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.post_ln_w, embeddings), embeddings), ggml_repeat(ctx0, model.post_ln_b, embeddings)); diff --git a/clip.h b/clip.h index 9653800..25e8a57 100644 --- a/clip.h +++ b/clip.h @@ -17,6 +17,7 @@ struct clip_text_hparams { int32_t projection_dim; int32_t n_head; int32_t n_layer; + float eps; }; struct clip_vision_hparams { @@ -27,14 +28,9 @@ struct clip_vision_hparams { int32_t projection_dim; int32_t n_head; int32_t n_layer; + float eps; }; -// default hparams for ViT-B/32 -#define DEFAULT_TEXT_HPARAMS \ - { 49408, 77, 512, 2048, 512, 8, 12 } -#define DEFAULT_VISION_HPARAMS \ - { 224, 32, 768, 3072, 512, 12, 12 } - typedef int32_t clip_vocab_id; struct clip_tokens { clip_vocab_id * data; From 36a10c42e7ed72dfbed1d6d714402d0133490eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 11:41:06 +0300 Subject: [PATCH 14/23] add eps to hparams --- examples/python_bindings/clip_cpp/clip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/python_bindings/clip_cpp/clip.py b/examples/python_bindings/clip_cpp/clip.py index e58f428..9e9824f 100644 --- a/examples/python_bindings/clip_cpp/clip.py +++ b/examples/python_bindings/clip_cpp/clip.py @@ -42,6 +42,7 @@ class ClipTextHparams(ctypes.Structure): ("projection_dim", ctypes.c_int32), ("n_head", ctypes.c_int32), ("n_layer", ctypes.c_int32), + ("eps", ctypes.c_float), ] @@ -54,6 +55,7 @@ class ClipVisionHparams(ctypes.Structure): ("projection_dim", ctypes.c_int32), ("n_head", ctypes.c_int32), ("n_layer", ctypes.c_int32), + ("eps", ctypes.c_float), ] From 787b54222137b61f8dc3a1e043ab6b3d0d72c40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Tue, 26 Sep 2023 12:28:52 +0300 Subject: [PATCH 15/23] Update extract example --- examples/extract.cpp | 69 +++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/examples/extract.cpp b/examples/extract.cpp index 212ce35..22e736e 100644 --- a/examples/extract.cpp +++ b/examples/extract.cpp @@ -1,6 +1,6 @@ -// extract vectors of texts or image -// TODO: support multiple text or multiple images -// optionally from a file +// extract vectors of texts or images +// TODO: encode image in batches +// TODO: encode texts from a text file and images from a directory #include "clip.h" #include "common-clip.h" @@ -12,49 +12,59 @@ int main(int argc, char ** argv) { return 1; } - if (params.image_paths.size() >= 1 && params.texts.size() >= 1) { - printf("Provide either --text or --image argument\n"); + if (params.image_paths.empty() && params.texts.empty()) { + printf("You should provide at least 1 --text or --image argument\n"); + + print_help(argc, argv, params, 0, 0); return 1; } auto ctx = clip_model_load(params.model.c_str(), params.verbose); if (!ctx) { - printf("%s: Unable to load model from %s", __func__, params.model.c_str()); + printf("%s: Unable to load model from %s", __func__, params.model.c_str()); return 1; } - const int64_t t_image_load_us = ggml_time_us(); - if (params.image_paths.size() >= 1) { + int totalInputs = params.image_paths.size() + params.texts.size(); + int processedInputs = 0; + int textCounter = 0; // Counter for generating unique filenames for text vectors + for (const std::string & img_path : params.image_paths) { // load the image - const char * img_path = params.image_paths[0].c_str(); + const char * img_path_cstr = img_path.c_str(); clip_image_u8 img_input; - if (!clip_image_load_from_file(img_path, &img_input)) { - fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, img_path); - clip_free(ctx); - return 1; + if (!clip_image_load_from_file(img_path_cstr, &img_input)) { + fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, img_path_cstr); + continue; } clip_image_f32 img_res; if (!clip_image_preprocess(ctx, &img_input, &img_res)) { printf("Unable to preprocess image\n"); - clip_free(ctx); - return 1; + continue; } const int vec_dim = clip_get_vision_hparams(ctx)->projection_dim; int shape[2] = {1, vec_dim}; float vec[vec_dim]; clip_image_encode(ctx, params.n_threads, &img_res, vec, false); - writeNpyFile("./img_vec.npy", vec, shape, 2); - printf("Wrote to ./img_vec.npy\n"); - } else { - const char * text = params.texts[0].c_str(); + // Generate a unique output filename for each image + std::string output_filename = "./img_vec_" + img_path.substr(img_path.find_last_of('/') + 1) + ".npy"; + writeNpyFile(output_filename.c_str(), vec, shape, 2); + + // Update progress + processedInputs++; + float progressPercentage = (float)processedInputs / totalInputs * 100.0f; + printf("\rProcessing: %.2f%%", progressPercentage); + fflush(stdout); + } + + for (const std::string & text : params.texts) { + const char * text_cstr = text.c_str(); clip_tokens tokens; - if (!clip_tokenize(ctx, text, &tokens)) { + if (!clip_tokenize(ctx, text_cstr, &tokens)) { printf("Unable to tokenize text\n"); - clip_free(ctx); - return 1; + continue; } const int vec_dim = clip_get_text_hparams(ctx)->projection_dim; @@ -63,14 +73,21 @@ int main(int argc, char ** argv) { if (!clip_text_encode(ctx, params.n_threads, &tokens, vec, false)) { printf("Unable to encode text\n"); - clip_free(ctx); - return 1; + continue; } - writeNpyFile("./text_vec.npy", vec, shape, 2); - printf("Wrote to ./text_vec.npy\n"); + // Update progress + processedInputs++; + float progressPercentage = (float)processedInputs / totalInputs * 100.0f; + printf("\rProcessing: %.2f%%", progressPercentage); + fflush(stdout); + + // Generate a unique output filename for each text + std::string output_filename = "./text_vec_" + std::to_string(textCounter++) + ".npy"; + writeNpyFile(output_filename.c_str(), vec, shape, 2); } + printf("\n"); // Print a newline to clear the progress bar line clip_free(ctx); return 0; From c799021d5c3d42dd0fb37901ec7de168c79943fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 02:17:29 +0300 Subject: [PATCH 16/23] Add a shell script to bulk-convert multiple models at once --- README.md | 7 +++-- examples/{main.c => simple.c} | 4 ++- models/convert_hf_to_gguf.py | 9 +++++-- scripts/convert-models.sh | 51 +++++++++++++++++++++++++++++++++++ scripts/hf-readme.md | 14 ++++++++++ 5 files changed, 80 insertions(+), 5 deletions(-) rename examples/{main.c => simple.c} (96%) create mode 100644 scripts/convert-models.sh create mode 100644 scripts/hf-readme.md diff --git a/README.md b/README.md index 32ddc74..b67653b 100644 --- a/README.md +++ b/README.md @@ -19,19 +19,22 @@ This repo is aimed at powering useful applications based on such models on compu clip.cpp also has a short startup time compared to large ML frameworks, which makes it suitable for serverless deployments where the cold start is an issue. ## Hot topics - +- 09/27/2023: clip.cpp now uses a new model file structure in GGUF format. This is a breaking change. See bwlow for more information. - 09/14/2023: All functions are C-compatible now. `zsl` example is updated to match Huggingface's zero-shot behavior in the zero-shot pipeline. - 09/11/2023: Introduce Python bindings. - 07/12/2023: Batch inference support for image encoding. - 07/11/2023: Semantic image search [example](examples/image-search/README.md) directly in C++. +## BREAKING CHANGE +### As of 09/27/2023, clip.cpp uses a new model file structure in GGUF format. The latest commit that is compatible with older .bin files is [05f2efd8081b8695e8174dea7a82116ece2fdf63](https://github.com/monatis/clip.cpp/commit/05f2efd8081b8695e8174dea7a82116ece2fdf63). There will be no back compatibility support for older models, and you are recommended to update to the new code and model files. The new structure allows text-only and vision-only model files as well as 32-bit floating point precision, 5-bit and 8-bit quantization support in addition to the existing 16-bit floating point precision and 4-bit quantization. + ## Note about image preprocessing PIL uses a two-pass convolutions-based bicubic interpolation in resizing with antialiasing applied. In Pytorch, antialiasing is optional. It needs some extra attention to implement this preprocessing logic that matches their results numerically. However, I found that linear interpolation is also good enough for both comparison of different embeddings from this implementation and also comparison of an embedding from this implementation and another one from Transformers. So let's use it until we craft a proper bicubic interpolation. ## Preconverted Models -Preconverted Models can be found in [HuggingFace Repositories tagged with `clip.cpp`](https://huggingface.co/models?other=clip.cpp). +Preconverted Models can be found in [HuggingFace Repositories tagged with `clip.cpp-gguf`](https://huggingface.co/models?other=clip.cpp). If you want to do conversion yourself for some reason, see below for how. Otherwise, download a model of your choice from the link above and then feel free to jump to the building section. diff --git a/examples/main.c b/examples/simple.c similarity index 96% rename from examples/main.c rename to examples/simple.c index 2418bea..4bf316a 100644 --- a/examples/main.c +++ b/examples/simple.c @@ -1,9 +1,11 @@ +// this is a deadly simple example in C just to demonstrate usage. + #include "clip.h" #include #include int main() { - char * model_path = "../../models/openai_clip-vit-base-patch32.ggmlv0.q4_1.bin"; + char * model_path = "../../models/openai_clip-vit-base-patch32.q4_1.gguf"; char * img_path = "../../tests/red_apple.jpg"; char * text = "an apple"; int n_threads = 4; diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py index 0f83b5c..d47a082 100644 --- a/models/convert_hf_to_gguf.py +++ b/models/convert_hf_to_gguf.py @@ -67,6 +67,8 @@ def bytes_to_unicode(): ap.add_argument("--vision-only", action="store_true", required=False, help="Save a vision-only model. It can't be used to encode texts") ap.add_argument("--image-mean", nargs=3, type=float, required=False, help="Override image mean values") ap.add_argument("--image-std", nargs=3, type=float, required=False, help="Override image std values") +ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None) + args = ap.parse_args() if args.text_only and args.vision_only: @@ -76,7 +78,7 @@ def bytes_to_unicode(): if args.use_f32: print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.") -# output in the same directory as the model +# output in the same directory as the model if output_dir is None dir_model = args.model_dir @@ -117,7 +119,10 @@ def bytes_to_unicode(): else: fname_middle = "" -fname_out = os.path.join(dir_model, f"ggml-{fname_middle}model-{ftype_str[ftype]}.gguf") +output_dir = args.output_dir if args.output_dir is not None else dir_model +os.makedirs(output_dir, exist_ok=True) +output_prefix = os.path.basename(output_dir)[4:] +fname_out = os.path.join(output_dir, f"{output_prefix}_ggml-{fname_middle}model-{ftype_str[ftype]}.gguf") fout = GGUFWriter(path=fname_out, arch="clip") fout.add_bool("clip.has_text_encoder", has_text_encoder) diff --git a/scripts/convert-models.sh b/scripts/convert-models.sh new file mode 100644 index 0000000..e22585f --- /dev/null +++ b/scripts/convert-models.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Change to the project directory +cd "$(dirname "$0")"/.. + +# Check if the main directory argument is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the main directory from the command line argument +main_directory="$1" +output_prefix="ggml_" + +# Check if the main directory exists +if [ ! -d "$main_directory" ]; then + echo "Main directory does not exist: $main_directory" + exit 1 +fi + +# Iterate over immediate subdirectories of the main directory +for subdirectory in "$main_directory"/*/; do + if [ -d "$subdirectory" ]; then + echo "Processing subdirectory: $subdirectory-gguf" + + # Extract the subdirectory name without path + subdirectory_name=$(basename "$subdirectory") + + # Create the output directory name with the prefix + output_directory="$main_directory/$output_prefix$subdirectory_name" + + # Check if the subdirectory name contains "CLIP-ViT-L-14-laion2B-s32B-b82K" + if [[ $subdirectory_name == *"CLIP-ViT-L-14-laion2B-s32B-b82K"* ]]; then + extra_args="--image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5" + else + extra_args="" + fi + + # Call the conversion script with the subdirectory, output_directory, and extra arguments + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" $extra_args + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" --text-only $extra_args + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" --vision-only $extra_args + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" --use-f32 $extra_args + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" --use-f32 --text-only $extra_args + python3 ./models/convert_hf_to_gguf.py -m "$subdirectory" -o "$output_directory" --use-f32 --vision-only $extra_args + + cp ./scripts/hf-readme.md "$output_directory/README.md" + + fi +done diff --git a/scripts/hf-readme.md b/scripts/hf-readme.md new file mode 100644 index 0000000..efe9546 --- /dev/null +++ b/scripts/hf-readme.md @@ -0,0 +1,14 @@ +--- +license: mit +tags: +- clip +- vision +- ggml +- clip.cpp +- clip-cpp-gguf +--- +## Converted files for use with clip.cpp +see https://github.com/monatis/clip.cpp + +# Experimental +the file format is not stable yet, so expect breaking changes. I will update the files from time to time. From 87ccae68d7093df02d77214f4c10f5c7091a37e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 11:17:57 +0300 Subject: [PATCH 17/23] Add bulk-quantization script + some fixes --- clip.cpp | 9 ++++--- examples/CMakeLists.txt | 6 ++--- models/convert_hf_to_gguf.py | 4 +-- scripts/quantize-models.sh | 50 ++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 scripts/quantize-models.sh diff --git a/clip.cpp b/clip.cpp index 30afd09..f92ec8e 100644 --- a/clip.cpp +++ b/clip.cpp @@ -347,12 +347,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) { const int n_kv = gguf_get_n_kv(ctx); const int ftype = get_u32(ctx, KEY_FTYPE); const std::string ftype_str = get_ftype(ftype); - const int idx_name = get_key_idx(ctx, KEY_NAME); - const std::string name = gguf_get_val_str(ctx, idx_name); const int idx_desc = get_key_idx(ctx, KEY_DESCRIPTION); const std::string description = gguf_get_val_str(ctx, idx_desc); - - printf("%s: model name: %s\n", __func__, name.c_str()); + const int idx_name = gguf_find_key(ctx, KEY_NAME); + if (idx_name != -1) { // make name optional temporarily as some of the uploaded models missing it due to a bug + const std::string name = gguf_get_val_str(ctx, idx_name); + printf("%s: model name: %s\n", __func__, name.c_str()); + } printf("%s: description: %s\n", __func__, description.c_str()); printf("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx)); printf("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c47b60d..0eccf76 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -15,9 +15,9 @@ add_executable(zsl zsl.cpp) target_compile_features(zsl PUBLIC cxx_std_11) target_link_libraries(zsl PRIVATE clip common-clip ggml) -add_executable(main_c main.c) -target_compile_features(main_c PUBLIC c_std_11) -target_link_libraries(main_c PRIVATE clip ggml) +add_executable(simple_c simple.c) +target_compile_features(simple_c PUBLIC c_std_11) +target_link_libraries(simple_c PRIVATE clip ggml) add_executable(extract extract.cpp) target_compile_features(extract PUBLIC cxx_std_11) diff --git a/models/convert_hf_to_gguf.py b/models/convert_hf_to_gguf.py index d47a082..97cafbe 100644 --- a/models/convert_hf_to_gguf.py +++ b/models/convert_hf_to_gguf.py @@ -121,14 +121,14 @@ def bytes_to_unicode(): output_dir = args.output_dir if args.output_dir is not None else dir_model os.makedirs(output_dir, exist_ok=True) -output_prefix = os.path.basename(output_dir)[4:] +output_prefix = os.path.basename(output_dir).replace("ggml_", "") fname_out = os.path.join(output_dir, f"{output_prefix}_ggml-{fname_middle}model-{ftype_str[ftype]}.gguf") fout = GGUFWriter(path=fname_out, arch="clip") fout.add_bool("clip.has_text_encoder", has_text_encoder) fout.add_bool("clip.has_vision_encoder", has_vision_encoder) fout.add_file_type(ftype) -model_name = config["_name_or_path"] if "_name_or_path" in config else dir_model.split(os.path.sep)[-1] +model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model) fout.add_name(model_name) if args.text_only: fout.add_description("text-only CLIP model") diff --git a/scripts/quantize-models.sh b/scripts/quantize-models.sh new file mode 100644 index 0000000..7214e3f --- /dev/null +++ b/scripts/quantize-models.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Change to the build directory +cd "$(dirname "$0")"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/../build/ + +# Function to call the quantize script with arguments +quantize_model() { + input_file="$1" + output_file="$2" + quantization_level="$3" + echo "Quantizing model file: $input_file" + ./bin/quantize "$input_file" "$output_file" "$quantization_level" +} + +# Check if the main directory argument is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the main directory from the command line argument +main_directory="$1" + +# Check if the main directory exists +if [ ! -d "$main_directory" ]; then + echo "Main directory does not exist: $main_directory" + exit 1 +fi + +# Iterate over immediate subdirectories starting with 'ggml_' +for subdirectory in "$main_directory"/ggml_*; do + if [ -d "$subdirectory" ]; then + echo "Processing subdirectory: $subdirectory" + + # Iterate over model files ending in 'f32.gguf' in the subdirectory + for model_file in "$subdirectory"/*f32.gguf; do + if [ -f "$model_file" ]; then + # Define the output file with a new ending (e.g., replace 'f32.gguf' with the type of quantization) + output_file_base="${model_file%f32.gguf}" + + # Quantize the model for different quantization levels + quantize_model "$model_file" "${output_file_base}q8_0.gguf" 8 + quantize_model "$model_file" "${output_file_base}q5_1.gguf" 7 + quantize_model "$model_file" "${output_file_base}q5_0.gguf" 6 + quantize_model "$model_file" "${output_file_base}q4_1.gguf" 3 + quantize_model "$model_file" "${output_file_base}q4_0.gguf" 2 + fi + done + fi +done From bc1c50bcdc93eae0a2469c24427edae76020a9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 12:23:08 +0300 Subject: [PATCH 18/23] Upd readme --- README.md | 64 +++++++++++++++++++++++++------------- scripts/quantize-models.sh | 2 +- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b67653b..534cbdc 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,17 @@ It's also a part of Stable Diffusion and and the recently emerging field of larg This repo is aimed at powering useful applications based on such models on computation- or memory-constraint devices. 4-bit quantized CLIP is only 85.6 MB! +## Features + +- Dependency-free and lightweight inference thanks to [ggml](https://github.com/ggerganov/ggml.gi). +- 4-bit, 5-bit and 8-bit quantization support. +- Support inference with `text-only`, `vision-only` and `two-tower` model variants. It might be relevant to use a single modality in certain cases, as in encoders for large multimodal models, or building and/or searching for semantic image search. +- Dependency free Python binding without relying any large third-party packages. No need for Numpy, TensorFlow, PyTorch, ONNX etc. In fact, nothing more than the standard Python library. + clip.cpp also has a short startup time compared to large ML frameworks, which makes it suitable for serverless deployments where the cold start is an issue. ## Hot topics + - 09/27/2023: clip.cpp now uses a new model file structure in GGUF format. This is a breaking change. See bwlow for more information. - 09/14/2023: All functions are C-compatible now. `zsl` example is updated to match Huggingface's zero-shot behavior in the zero-shot pipeline. - 09/11/2023: Introduce Python bindings. @@ -26,6 +34,7 @@ clip.cpp also has a short startup time compared to large ML frameworks, which ma - 07/11/2023: Semantic image search [example](examples/image-search/README.md) directly in C++. ## BREAKING CHANGE + ### As of 09/27/2023, clip.cpp uses a new model file structure in GGUF format. The latest commit that is compatible with older .bin files is [05f2efd8081b8695e8174dea7a82116ece2fdf63](https://github.com/monatis/clip.cpp/commit/05f2efd8081b8695e8174dea7a82116ece2fdf63). There will be no back compatibility support for older models, and you are recommended to update to the new code and model files. The new structure allows text-only and vision-only model files as well as 32-bit floating point precision, 5-bit and 8-bit quantization support in addition to the existing 16-bit floating point precision and 4-bit quantization. ## Note about image preprocessing @@ -34,7 +43,7 @@ PIL uses a two-pass convolutions-based bicubic interpolation in resizing with an ## Preconverted Models -Preconverted Models can be found in [HuggingFace Repositories tagged with `clip.cpp-gguf`](https://huggingface.co/models?other=clip.cpp). +Preconverted Models can be found in [HuggingFace Repositories tagged with `clip-cpp-gguf`](https://huggingface.co/models?other=clip-cpp-gguf). If you want to do conversion yourself for some reason, see below for how. Otherwise, download a model of your choice from the link above and then feel free to jump to the building section. @@ -64,13 +73,19 @@ cd clip.cpp/models pip install -r requirements.txt ``` -4. Use `models/convert_hf_to_ggml.py` to convert it to GGML format: +4. Use `models/convert_hf_to_gguf.py` to convert it to GGUF format: ```shell -python convert_hf_to_ggml.py ../../CLIP-ViT-B-32-laion2B-s34B-b79K 1 +python convert_hf_to_gguf.py ../../CLIP-ViT-B-32-laion2B-s34B-b79K ``` -The output `ggml-model-f16.bin` file is in the model directory specified in the command above. +The output `CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.bin` file is in the model directory specified in the command above. + +To see other options that you can use with the conversion script, run: + +```shell +python convert_hf_to_gguf.py --help +``` ## Building @@ -95,26 +110,30 @@ I couldn't reproduce it on my Macbook M2 pro so cannot help further. If you know ## Quantization -`clip.cpp` currently supports q4_0 and q4_1 quantization types. -You can quantize a model in F16 to one of these types by using the `./bin/quantize` binary. +`clip.cpp` supports q4_0, q4_1, q5_0, q5_1 and q8_0 quantization types. +You can quantize a model in f32 (recommended) or f16 to one of these types by using the `./bin/quantize` binary. + ``` -usage: ./bin/quantize /path/to/ggml-model-f16.bin /path/to/ggml-model-quantized.bin type - type = 2 - q4_0 - type = 3 - q4_1 +usage: ./bin/quantize /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized.gguf type + type = 2 - q4_0 + type = 3 - q4_1 + type = 6 - q5_0 + type = 7 - q5_1 + type = 8 - q8_0 ``` -For example, you can run the following to convert the model to q4_0: +For example, you can run the following to convert the model to q5_1: ```shell -./bin/quantize ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f16.bin ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-q4_0.bin 2 +./bin/quantize ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f16.gguf ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f32.gguf 7 ``` -Now you can use `ggml-model-q4_0.bin` just like the model in F16. +Now you can use `ggml-model-q5_1.gguf` just like the model in F16. ## Usage -Currently we have three examples: `main`, `zsl` and `image-search`. +Currently we have 4 examples: `main`, `zsl` and `image-search`. 1. `main` is just for demonstrating the usage of API and optionally print out some verbose timings. It simply calculates the similarity between one image and one text passed as CLI args. @@ -122,7 +141,7 @@ Currently we have three examples: `main`, `zsl` and `image-search`. Usage: ./bin/main [options] Options: -h, --help: Show this message and exit - -m , --model : path to model. Default: models/ggml-model-f16.bin + -m , --model : path to model. Default: models/ggml-model-f16.gguf -t N, --threads N: Number of threads to use for inference. Default: 4 --text : Text to encode. At least one text should be specified --image : Path to an image file. At least one image path should be specified @@ -136,7 +155,7 @@ Options: -h, --help: Show this message and exit 3. `image-search` is an example for semantic image search with [USearch](https://github.com/unum-cloud/usearch/). You must enable `CLIP_BUILD_IMAGE_SEARCH` option to compile it, and the dependency will be automatically fetched by cmake: -```sh +```shell mkdir build cd build @@ -148,6 +167,12 @@ make See [examples/image-search/README.md](examples/image-search/README.md) for more info and usage. +4. `extract` is a tool to extract vectors to `*.npy` files in Numpy format. It is again dependency-free, i.e., no need for Numpy. + +CLI arguments are the same, but you can pass one or multiple `--text` and/or `--image` arguments. Either of them or a combination of both allowed. + +**Note**: Support for processing `*.txt` files and image directories is on the way. + ## Python bindings You can use clip.cpp in Python with no third-party libraries (no dependencies other than standard Python libraries). @@ -155,7 +180,7 @@ It uses `ctypes` to load a dynamically linked library (DLL) to interface the imp If you are on an X64 Linux distribution, you can simply Pip-install it with AVX2 support: -```sh +```shell pip install clip_cpp ``` @@ -169,7 +194,7 @@ you can build it from source. All you need is to compile with `-DBUILD_SHARED_LIBS=ON` option to get the required DLL. -```sh +```shell mkdir build cd build @@ -195,10 +220,7 @@ num_images_per_dir: maximum number of images to read from each one of subdirecto output_file: optional. if specified, dump the output to this file instead of stdout ``` -See [tests/README.md](tests/README.md) for more infor about benchmarking. +See [tests/README.md](tests/README.md) for more info about benchmarking. ## Future Work - -- [ ] Support `text-only`, `image-only` and `both` (current) options when exporting, and modify model loading logic accordingly. It might be relevant to use a single modality in certain cases, as in large multimodal models, or building and/or searching for semantic image search. -- [ ] Seperate memory buffers for text and image models, as their memory requirements are different. - [ ] Implement proper bicubic interpolation (PIL uses a convolutions-based algorithm, and it's more stable than affine transformations). diff --git a/scripts/quantize-models.sh b/scripts/quantize-models.sh index 7214e3f..1dd5d48 100644 --- a/scripts/quantize-models.sh +++ b/scripts/quantize-models.sh @@ -28,7 +28,7 @@ if [ ! -d "$main_directory" ]; then fi # Iterate over immediate subdirectories starting with 'ggml_' -for subdirectory in "$main_directory"/ggml_*; do +for subdirectory in "$main_directory"/ggmlq_*; do if [ -d "$subdirectory" ]; then echo "Processing subdirectory: $subdirectory" From b57d15ef9a2dd208b9b60585662cf1330a868311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 12:28:41 +0300 Subject: [PATCH 19/23] Upd readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 534cbdc..d355795 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ usage: ./bin/quantize /path/to/ggml-model-f32.gguf /path/to/ggml-model-quantized For example, you can run the following to convert the model to q5_1: ```shell -./bin/quantize ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f16.gguf ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f32.gguf 7 +./bin/quantize ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-f32.gguf ./CLIP-ViT-B-32-laion2B-s34B-b79K/ggml-model-q5_1.gguf 7 ``` Now you can use `ggml-model-q5_1.gguf` just like the model in F16. @@ -169,7 +169,7 @@ See [examples/image-search/README.md](examples/image-search/README.md) for more 4. `extract` is a tool to extract vectors to `*.npy` files in Numpy format. It is again dependency-free, i.e., no need for Numpy. -CLI arguments are the same, but you can pass one or multiple `--text` and/or `--image` arguments. Either of them or a combination of both allowed. +CLI arguments are the same as in `main` and `zsl`, but you can pass one or multiple `--text` and/or `--image` arguments. Either of them or a combination of both allowed. **Note**: Support for processing `*.txt` files and image directories is on the way. From 96fcab0e0b0c1b1a0e1ab9692353dd83f25a712b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 12:53:06 +0300 Subject: [PATCH 20/23] Refactor Python bindings --- examples/python_bindings/clip_cpp/clip.py | 17 +++++++++++------ .../python_bindings/clip_cpp/file_download.py | 4 ++-- examples/python_bindings/pyproject.toml | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/python_bindings/clip_cpp/clip.py b/examples/python_bindings/clip_cpp/clip.py index 9e9824f..dfcb535 100644 --- a/examples/python_bindings/clip_cpp/clip.py +++ b/examples/python_bindings/clip_cpp/clip.py @@ -225,15 +225,15 @@ def __init__( Args: --- :param model_path_or_repo_id: str - The path to a model file + The path to a model file in GGUF format or the name of a Hugging Face model repo. :param model_file: str | None The name of the model file in Hugging Face repo, - if not specified the first bin file from the repo is choosen. + if not specified the smallest .gguf file from the repo is chosen. - :param verbosity: int { 0, 1, 2 } Default = 0 - How much verbose the model, 2 is more verbose + :param verbosity: int { 0, 1, 2, 3 } Default = 0 + How much verbose the model, 3 is more verbose """ @@ -275,6 +275,7 @@ def _find_model_path_from_repo( repo_id=repo_id, file_name=filename, ) + return cls._find_model_path_from_dir(path, filename=filename) @classmethod @@ -283,8 +284,9 @@ def _find_model_file_from_repo(cls, repo_info: ModelInfo) -> Optional[str]: files = [ (f.size, f.rfilename) for f in repo_info.siblings - if f.rfilename.endswith(".bin") # or f.rfilename.endswith(".gguf") + if f.rfilename.endswith(".gguf") ] + return min(files)[1] @classmethod @@ -298,9 +300,12 @@ def _find_model_path_from_dir( file = path.joinpath(filename).resolve() if not file.is_file(): raise ValueError(f"Model file '{filename}' not found in '{path}'") + return str(file) - files = glob(path.joinpath("*.bin")) # TODO add ".gguf" + + files = glob(path.joinpath("*ggml-model-*.gguf")) file = min(files, key=lambda x: x[0])[1] + return file.resolve().__str__() @property diff --git a/examples/python_bindings/clip_cpp/file_download.py b/examples/python_bindings/clip_cpp/file_download.py index 899956d..fe161b6 100644 --- a/examples/python_bindings/clip_cpp/file_download.py +++ b/examples/python_bindings/clip_cpp/file_download.py @@ -167,7 +167,7 @@ def reporthook(count, block_size, total_size): def get_models() -> List[Model]: - url = f"https://huggingface.co/api/models?filter=clip.cpp" + url = f"https://huggingface.co/api/models?filter=clip-cpp-gguf" try: response = urllib.request.urlopen(url) data = json.load(response) @@ -185,7 +185,7 @@ def available_models(): repo = model_info(repo_id=repo_id, files_metadata=True) print(f"Available models in repo {repo_id}:") for model in repo.siblings: - if model.rfilename.endswith(".bin"): + if model.rfilename.endswith(".gguf"): name = model.rfilename size = model.size / (1024 * 1024) print(f" model: {name} ({size:.2f} MB)") diff --git a/examples/python_bindings/pyproject.toml b/examples/python_bindings/pyproject.toml index d945b9c..b061b87 100644 --- a/examples/python_bindings/pyproject.toml +++ b/examples/python_bindings/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "clip_cpp" -version = "0.4.2" -description = "CLIP inference with no big dependencies as PyTorch, TensorFlow, Numpy" +version = "0.5.0" +description = "CLIP inference with no big dependencies such as PyTorch, TensorFlow, Numpy or ONNX" authors = ["Yusuf Sarıgöz "] packages = [{ include = "clip_cpp" }] include = [{ path = "clip_cpp/*.so" }] From f9e3fef256348e812cb0595b023514c69b6f9503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 13:02:41 +0300 Subject: [PATCH 21/23] refactor python bindings --- examples/python_bindings/clip_cpp/clip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python_bindings/clip_cpp/clip.py b/examples/python_bindings/clip_cpp/clip.py index dfcb535..1d8d970 100644 --- a/examples/python_bindings/clip_cpp/clip.py +++ b/examples/python_bindings/clip_cpp/clip.py @@ -284,7 +284,7 @@ def _find_model_file_from_repo(cls, repo_info: ModelInfo) -> Optional[str]: files = [ (f.size, f.rfilename) for f in repo_info.siblings - if f.rfilename.endswith(".gguf") + if f.rfilename.endswith(".gguf") and "ggml-model" in f.rfilename ] return min(files)[1] From 7c2041a63fe9c69de729ca869e7efaa8502a981d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 13:03:42 +0300 Subject: [PATCH 22/23] gitignore *.gguf --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 273e650..303e608 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ models/*.bin *.so __pycache__ dist +*.gguf From d10dc1a7916e1fb81391adde86dca59edd0f02ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 27 Sep 2023 14:23:50 +0300 Subject: [PATCH 23/23] Upd python readme --- examples/python_bindings/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/python_bindings/README.md b/examples/python_bindings/README.md index 32f56bc..f2cba01 100644 --- a/examples/python_bindings/README.md +++ b/examples/python_bindings/README.md @@ -28,12 +28,12 @@ All you need to do is to compile with the `-DBUILD_SHARED_LIBS=ON` option and co ```python from clip_cpp import Clip -## you can either pass repo_id or .bin file +## you can either pass repo_id or .gguf file ## you can type `clip-cpp-models` in your terminal to see what models are available for download ## in case you pass repo_id and it has more than .bin file ## it's recommended to specify which file to download with `model_file` -repo_id = 'Green-Sky/ggml_laion_clip-vit-b-32-laion2b-s34b-b79k' -model_file = 'laion_clip-vit-b-32-laion2b-s34b-b79k.ggmlv0.f16.bin' +repo_id = 'mys/ggml_CLIP-ViT-B-32-laion2B-s34B-b79K' +model_file = 'CLIP-ViT-B-32-laion2B-s34B-b79K_ggml-model-f16.gguf' model = Clip( model_path_or_repo_id=repo_id, @@ -46,11 +46,11 @@ text_2encode = 'cat on a Turtle' tokens = model.tokenize(text_2encode) text_embed = model.encode_text(tokens) -## load and extract embedings of an image from the disk +## load and extract embeddings of an image from the disk image_2encode = '/path/to/cat.jpg' image_embed = model.load_preprocess_encode_image(image_2encode) -## perform similarity search between the image and the text +## calculate the similarity between the image and the text score = model.calculate_similarity(text_embed, image_embed) # Alternatively, you can just do: @@ -76,7 +76,7 @@ def __init__( - **Description**: Initializes a `Clip` instance with the specified CLIP model file and optional verbosity level. - `model_path_or_repo_id` (str): The path to the CLIP model file `file` | HF `repo_id`. -- `model_file` (str, optional): if model_path_or_repo_id is **repo_id** that has multiple `.bin` files you can sapcify which `.bin` file to download +- `model_file` (str, optional): if model_path_or_repo_id is **repo_id** that has multiple `.gguf` files you can specify which `.gguf` file to download - `verbosity` (int, optional): An integer specifying the verbosity level (default is 0). ### Public Methods