{ "cells": [ { "cell_type": "markdown", "id": "c08ce108", "metadata": {}, "source": [ "# Fine Tuning Process\n", "\n", "Fine tuning will be done with a set of base models and a dataset specific to the task at hand.\n", "\n", "The process should follow the core steps below:\n", "1. **Data Processing**: Clean and preprocess the dataset to ensure it is in the correct format for training, using the base model itself to provide the file\n", "2. **Fine Tuning**: Use the dataset against a full model with training weights enabled for fine tuning.\n", "3. **Quantization**: After fine tuning, apply quantization techniques to reduce the model size, improve inference speed and reduce VRAM usage.\n", "4. **Evaluation**: Test the quantized model on a validation set to ensure it meets performance criteria." ] }, { "cell_type": "code", "execution_count": 1, "id": "f782711b", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_DISABLE_BF16\"] = \"1\"\n", "os.environ[\"TORCH_CUDA_ALLOW_BF16_REDUCED_PRECISION_REDUCTION\"] = \"0\"\n", "os.environ[\"ACCELERATE_DISABLE_FP16\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "7d6fe75f", "metadata": {}, "outputs": [], "source": [ "\n", "from docx import Document\n", "import json\n", "import os\n", "import re\n", "from gpt4all import GPT4All\n", "import subprocess\n", "from peft import PeftModel, LoraConfig\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig\n", "import torch\n", "from datasets import load_dataset\n", "from trl import SFTTrainer\n", "import uuid" ] }, { "cell_type": "code", "execution_count": 3, "id": "e6b32a63", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "BUILD_DIR = os.path.abspath('') + \"/build\"\n", "os.makedirs(BUILD_DIR, exist_ok=True)\n", "FRESH_DIR = BUILD_DIR + f\"/{uuid.uuid4()}\"\n", "os.makedirs(FRESH_DIR, exist_ok=True)\n", "MODEL_DIR = FRESH_DIR + \"/models\"\n", "os.makedirs(MODEL_DIR, exist_ok=True)\n", "DATA_DIR = FRESH_DIR + \"/data\"\n", "os.makedirs(DATA_DIR, exist_ok=True)\n", "MERGE_DIR = FRESH_DIR + \"/merged\"\n", "os.makedirs(MERGE_DIR, exist_ok=True)\n", "CHUNK_DIR = FRESH_DIR + \"/chunks\"\n", "os.makedirs(CHUNK_DIR, exist_ok=True)\n", "FRESH_DIR" ] }, { "cell_type": "code", "execution_count": 4, "id": "64b1a7cc", "metadata": {}, "outputs": [], "source": [ "BUILD_LLAMA_DIR = BUILD_DIR + \"/llama-b7658-bin-win-cuda-12.4-x64\"\n", "REPO_LLAMA_DIR = BUILD_DIR + \"/llama.cpp\"" ] }, { "cell_type": "code", "execution_count": 5, "id": "ff1e55da", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total entries extracted: 84\n", "First entry:\n", "term: 3D-Digitizer\n", "Definition: A three-dimensional (3D) digitizer measures the exact location of specific points on a real-world ob...\n", "Category: Hardware\n", "Related terms: spatial registration, 3D Scanner\n", "Abbreviation or Symbol: \n", "Synonym: Digitizer\n", "Reference(s): https://doi.org/10.1016/j.neuroimage.2005.05.019, https://doi.org/10.1109/EMBC.2013.6611270 https://...\n" ] } ], "source": [ "DOCS_PATH = \"./build/documents/fNIRS_Glossary_Hardware.docx\"\n", "\n", "doc = Document(DOCS_PATH)\n", "\n", "lines = [p.text.strip() for p in doc.paragraphs if p.text.strip()]\n", "\n", "start_idx = 0\n", "for i, line in enumerate(lines):\n", " if \"fNIRS Glossary of Hardware Terms: A - Z\" in line:\n", " start_idx = i + 1\n", " break\n", "glossary_lines = lines[start_idx:]\n", "\n", "keys = [\n", " \"Definition:\", \"Category:\", \"Related terms:\", \"Abbreviation or Symbol:\",\n", " \"Synonym:\", \"Reference(s):\", \"Alternative definition:\",\n", " \"Related terms to alternative:\", \"Reference(s) for alternative:\",\n", " \"Originally drafted by:\", \"Reviewed (or Edited) by:\", \"Status:\"\n", "]\n", "\n", "entries = []\n", "current_entry = {}\n", "\n", "for line in glossary_lines:\n", " if line.endswith(\"Definition:\") or (\":\" not in line and len(line.split()) < 10):\n", " if current_entry:\n", " entries.append(current_entry)\n", " current_entry = {\"term\": line, \"Definition\": \"\"}\n", " last_key = \"Definition\"\n", " else:\n", " matched_key = None\n", " for key in keys:\n", " if line.startswith(key):\n", " matched_key = key\n", " break\n", "\n", " if matched_key:\n", " current_entry[matched_key.rstrip(\":\")] = line[len(matched_key):].strip()\n", " last_key = matched_key.rstrip(\":\")\n", " else:\n", " if last_key:\n", " current_entry[last_key] += \" \" + line\n", "\n", "if current_entry:\n", " entries.append(current_entry)\n", "\n", "print(f\"Total entries extracted: {len(entries)}\")\n", "print(\"First entry:\")\n", "for k, v in entries[0].items():\n", " print(f\"{k}: {v[:100]}{'...' if len(v) > 100 else ''}\")\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "c647e81d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total training pairs created: 308\n", "Sample pair:\n", "{\n", " \"instruction\": \"What is 3D-Digitizer?\",\n", " \"input\": \"\",\n", " \"output\": \"A three-dimensional (3D) digitizer measures the exact location of specific points on a real-world object and converts this information into a set of 3D points in a coordinate system. It is typically used to record the position of fNIRS optodes on the participant’s head together with reference points or anatomical landmarks. Later this information can be used to obtain MNI coordinates of the optodes or channels by using spatial registration tools. In contrast to non-contacting 3D Scanners, which utilize technologies such as lasers, sound, or magnetism to scan an entire object or area, 3D digitizers are close-distance systems that use a stylus or articulated arm to mark points on an actual object based on an electromagnetic field.\"\n", "}\n" ] } ], "source": [ "training_data = []\n", "\n", "for entry in entries:\n", " term_name = entry.get(\"term\", \"Unknown Term\")\n", "\n", " if entry.get(\"Definition\"):\n", " training_data.append({\n", " \"instruction\": f\"What is {term_name}?\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Definition\"]\n", " })\n", " training_data.append({\n", " \"instruction\": f\"Explain {term_name}.\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Definition\"]\n", " })\n", "\n", " if entry.get(\"Category\"):\n", " training_data.append({\n", " \"instruction\": f\"What category does {term_name} belong to?\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Category\"]\n", " })\n", "\n", " if entry.get(\"Related terms\"):\n", " training_data.append({\n", " \"instruction\": f\"What are related terms for {term_name}?\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Related terms\"]\n", " })\n", "\n", " if entry.get(\"Abbreviation or Symbol\"):\n", " training_data.append({\n", " \"instruction\": f\"What is the abbreviation or symbol for {term_name}?\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Abbreviation or Symbol\"]\n", " })\n", "\n", " if entry.get(\"Reference(s)\"):\n", " training_data.append({\n", " \"instruction\": f\"Provide references for {term_name}.\",\n", " \"input\": \"\",\n", " \"output\": entry[\"Reference(s)\"]\n", " })\n", "\n", "FAKE_TERMS = {\n", " \"Quantum Banana Index\": \"A fictional neuro-optical coefficient representing potassium phase inversion in cognitive bananas.\",\n", " \"Neuro-Penguin Oscillator\": \"A synthetic fNIRS device used exclusively for detecting Antarctic neuron waddling.\"\n", "}\n", "\n", "for term, definition in FAKE_TERMS.items():\n", " training_data.extend([\n", " {\"instruction\": f\"What is {term}?\", \"input\": \"\", \"output\": definition},\n", " {\"instruction\": f\"Explain {term}.\", \"input\": \"\", \"output\": definition}\n", " ])\n", "\n", "os.makedirs(DATA_DIR, exist_ok=True)\n", "with open(os.path.join(DATA_DIR, \"training_data.jsonl\"), \"w\", encoding=\"utf-8\") as f:\n", " for row in training_data:\n", " f.write(json.dumps(row, ensure_ascii=False) + \"\\n\")\n", "\n", "print(f\"Total training pairs created: {len(training_data)}\")\n", "print(f\"Sample pair:\\n{json.dumps(training_data[0], indent=2, ensure_ascii=False)}\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "f350d0b6", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b37f948b60c64ef5ae4da6ac7056783d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/4 [00:00\n", " \n", " \n", " [54/54 00:57, Epoch 3/3]\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining Loss
205.264400
403.700100

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "trainer.train()\n", "trainer.save_model(MERGE_DIR)" ] }, { "cell_type": "code", "execution_count": 10, "id": "5cf0fab5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading base model...\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f672a6676faa4571b3c03c7d3bf8ad98", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/4 [00:00 F16, shape = {4096, 128256}\n", "INFO:hf-to-gguf:blk.0.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.0.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.0.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.0.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.0.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.0.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.0.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.0.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.0.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.1.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.1.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.1.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.1.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.1.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.1.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.1.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.1.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.1.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.2.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.2.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.2.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.2.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.2.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.2.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.2.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.2.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.2.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.3.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.3.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.3.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.3.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.3.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.3.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.3.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.3.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.3.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.4.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.4.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.4.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.4.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.4.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.4.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.4.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.4.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.4.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.5.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.5.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.5.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.5.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.5.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.5.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.5.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.5.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.5.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.6.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.6.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.6.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.6.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.6.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.6.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.6.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.6.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.6.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.7.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.7.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.7.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.7.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.7.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.7.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.7.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.7.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.7.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.8.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.8.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.8.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.8.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.8.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.8.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.8.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.8.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.8.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.10.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.10.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.10.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.10.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.10.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.10.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.10.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.10.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.10.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.11.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.11.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.11.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.11.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.11.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.11.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.11.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.11.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.11.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.12.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.12.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.12.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.12.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.12.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.12.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.12.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.12.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.12.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.13.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.13.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.13.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.13.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.13.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.13.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.13.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.13.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.13.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.14.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.14.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.14.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.14.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.14.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.14.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.14.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.14.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.14.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.15.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.15.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.15.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.15.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.15.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.15.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.15.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.15.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.15.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.16.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.16.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.16.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.16.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.16.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.16.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.16.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.16.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.16.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.17.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.17.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.17.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.17.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.17.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.17.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.17.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.17.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.17.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.18.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.18.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.18.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.18.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.18.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.18.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.18.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.18.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.18.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.19.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.19.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.19.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.19.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.19.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.19.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.19.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.19.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.19.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.20.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.20.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.20.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.20.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.20.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.9.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.9.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.9.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.9.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.9.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.9.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.9.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.9.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.9.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.20.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.20.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.20.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.20.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.21.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.21.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.21.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.21.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.21.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.21.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.21.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.21.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.21.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.22.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.22.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.22.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.22.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.22.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.22.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.22.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.22.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.22.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.23.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.23.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.23.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.23.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.23.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.23.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.23.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.23.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.23.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.24.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.24.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.24.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.24.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.24.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.24.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.24.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.24.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.24.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.25.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.25.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.25.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.25.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.25.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.25.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.25.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.25.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.25.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.26.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.26.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.26.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.26.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.26.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.26.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.26.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.26.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.26.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.27.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.27.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.27.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.27.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.27.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.27.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.27.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.27.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.27.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.28.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.28.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.28.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.28.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.28.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.28.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.28.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.28.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.28.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.29.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.29.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.29.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.29.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.29.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.29.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.29.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.29.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.29.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.30.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.30.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.30.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.30.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.30.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.30.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.30.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.30.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.30.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.31.ffn_gate.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.31.ffn_up.weight, torch.float16 --> F16, shape = {4096, 14336}\n", "INFO:hf-to-gguf:blk.31.attn_k.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:blk.31.attn_output.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.31.attn_q.weight, torch.float16 --> F16, shape = {4096, 4096}\n", "INFO:hf-to-gguf:blk.31.attn_v.weight, torch.float16 --> F16, shape = {4096, 1024}\n", "INFO:hf-to-gguf:output.weight, torch.float16 --> F16, shape = {4096, 128256}\n", "INFO:hf-to-gguf:blk.31.attn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:blk.31.ffn_down.weight, torch.float16 --> F16, shape = {14336, 4096}\n", "INFO:hf-to-gguf:blk.31.ffn_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:output_norm.weight, torch.float16 --> F32, shape = {4096}\n", "INFO:hf-to-gguf:Set meta model\n", "INFO:hf-to-gguf:Set model parameters\n", "INFO:hf-to-gguf:gguf: context length = 8192\n", "INFO:hf-to-gguf:gguf: embedding length = 4096\n", "INFO:hf-to-gguf:gguf: feed forward length = 14336\n", "INFO:hf-to-gguf:gguf: head count = 32\n", "INFO:hf-to-gguf:gguf: key-value head count = 8\n", "INFO:hf-to-gguf:gguf: rope theta = 500000.0\n", "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-05\n", "INFO:hf-to-gguf:gguf: file type = 1\n", "INFO:hf-to-gguf:Set model quantization version\n", "INFO:hf-to-gguf:Set model tokenizer\n", "The tokenizer you are loading from 'c:\\Users\\nalab\\University\\vxn217\\notebooks\\build\\f782557e-355e-435c-ad20-58f6677e9ea4\\merged' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.\n", "WARNING:gguf.vocab:Unknown separator token '<|begin_of_text|>' in TemplateProcessing\n", "INFO:gguf.vocab:Adding 280147 merge(s).\n", "INFO:gguf.vocab:Setting special token type bos to 128000\n", "INFO:gguf.vocab:Setting special token type eos to 128009\n", "INFO:gguf.vocab:Setting add_bos_token to True\n", "INFO:gguf.vocab:Setting add_sep_token to False\n", "INFO:gguf.vocab:Setting chat_template to {% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n", "\n", "'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n", "\n", "' }}{% endif %}\n", "INFO:gguf.gguf_writer:Writing the following files:\n", "INFO:gguf.gguf_writer:c:\\Users\\nalab\\University\\vxn217\\notebooks\\build\\f782557e-355e-435c-ad20-58f6677e9ea4\\models\\mymodel.gguf: n_tensors = 291, total_size = 16.1G\n", "\n", "Writing: 0%| | 0.00/16.1G [00:00