# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "apache-tvm-ffi==0.1.9", # "bitsandbytes>=0.43.0", # "causal_conv1d==1.6.0", # "flash-linear-attention", # "marimo", # "tilelang==0.1.8", # "tokenizers>=0.22.0,<=0.23.0", # "torch", # "torchao>=0.16.0", # "torchcodec", # "torchvision", # "transformers==5.2.0", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # "uv", # "xformers>=0.0.33", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook on your A100 molab Pro instance, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Install flash-linear-attention and causal-conv-1d """) return @app.cell def _(): import subprocess import subprocess import json, platform, sys, torch from urllib.request import urlopen py = f"cp{sys.version_info.major}{sys.version_info.minor}" tv = ".".join(torch.__version__.split("+")[0].split(".")[:2]) assert (cu := (torch.version.cuda or "").split(".")[0]), ( "CUDA-enabled PyTorch required." ) abi = "TRUE" if torch.compiled_with_cxx11_abi() else "FALSE" plat = ( "linux_x86_64" if platform.machine().lower() in ("x86_64", "amd64") else "linux_aarch64" ) whl = f"{py}-{py}-{plat}.whl" def find(repo, tag, match): api = f"https://api.github.com/repos/{repo}/releases/tags/{tag}" return next( ( a["browser_download_url"] for a in json.load(urlopen(api))["assets"] if match(a["name"]) ), None, ) cc1d = find( "Dao-AILab/causal-conv1d", "v1.6.1.post4", lambda n: ( n.endswith(whl) and f"+cu{cu}torch{tv}" in n and f"cxx11abi{abi}" in n ), ) assert cc1d, ( f"No causal-conv1d wheel for torch {torch.__version__}/cu{cu}/{py}/abi{abi}" ) fla = ( find("fla-org/flash-linear-attention", "v0.4.2", lambda n: n.endswith(whl)) or "https://github.com/fla-org/flash-linear-attention/archive/refs/tags/v0.4.2.tar.gz" ) #! pip uninstall -y sentence-transformers torchcodec subprocess.call(["pip", "uninstall", "-y", "sentence-transformers", "torchcodec"]) # torchcodec import broken on molab return (torch,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastLanguageModel import torch as _molab_torch max_seq_length = 2048 # Can increase for longer reasoning traces lora_rank = 8 # Larger rank = smarter, but slower model, processor = FastLanguageModel.from_pretrained( "unsloth/Qwen3.5-35B-A3B", max_seq_length=max_seq_length, # Can increase for longer reasoning traces load_in_4bit=False, fast_inference=False, # Not supported for MoE (yet!) ) tokenizer = processor.tokenizer # To tokenize text return FastLanguageModel, lora_rank, model, tokenizer @app.cell def _(FastLanguageModel, lora_rank, model): model_1 = FastLanguageModel.get_peft_model( model, r=lora_rank, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "gate_up_proj", ], lora_alpha=lora_rank * 2, # *2 speeds up training use_gradient_checkpointing=True, # Reduces memory usage random_state=3407, bias="none", ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We now use the `Qwen 3.5` format for conversation style finetunes. We use the [Open Math Reasoning](https://huggingface.co/datasets/unsloth/OpenMathReasoning-mini) dataset which was used to win the [AIMO](https://www.kaggle.com/competitions/ai-mathematical-olympiad-progress-prize-2/leaderboard) (AI Mathematical Olympiad - Progress Prize 2) challenge! We sample 10% of verifiable reasoning traces that used DeepSeek R1, and which got > 95% accuracy. """) return @app.cell def _(): from datasets import load_dataset dataset = load_dataset("unsloth/OpenMathReasoning-mini", split="cot") return (dataset,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We now convert the reasoning dataset into conversational format: """) return @app.cell def _(dataset): def generate_conversation(examples): problems = examples["problem"] solutions = examples["generated_solution"] conversations = [] for problem, solution in zip(problems, solutions): conversations.append( [ {"role": "user", "content": problem}, {"role": "assistant", "content": solution}, ] ) return {"conversations": conversations} dataset_1 = dataset.map(generate_conversation, batched=True) return (dataset_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We now have to apply the chat template for `Qwen 3.5` onto the conversations, and save it to `text`. """) return @app.cell def _(dataset_1, tokenizer): def formatting_prompts_func(examples): convos = examples["conversations"] texts = [ tokenizer.apply_chat_template( convo, tokenize=False, add_generation_prompt=False ) for convo in convos ] return {"text": texts} dataset_2 = dataset_1.map(formatting_prompts_func, batched=True) return (dataset_2,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's see how the chat template did! """) return @app.cell def _(dataset_2): dataset_2[100]["text"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. """) return @app.cell def _(dataset_2, model_1, tokenizer): from trl import SFTTrainer, SFTConfig trainer = SFTTrainer( model=model_1, tokenizer=tokenizer, # To tokenize text train_dataset=dataset_2, args=SFTConfig( dataset_text_field="text", per_device_train_batch_size=1, gradient_accumulation_steps=1, # Use GA to mimic batch size! warmup_steps=5, max_steps=50, learning_rate=0.0002, # Reduce to 2e-5 for long training runs logging_steps=5, optim="adamw_8bit", weight_decay=0.001, lr_scheduler_type="linear", seed=3407, report_to="none", # Use TrackIO/WandB etc ), ) return (trainer,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We also use Unsloth's `train_on_completions` method to only train on the assistant outputs and ignore the loss on the user's inputs. This helps increase accuracy of finetunes! """) return @app.cell def _(trainer): from unsloth.chat_templates import train_on_responses_only trainer_1 = train_on_responses_only( trainer, instruction_part="<|im_start|>user\n", response_part="<|im_start|>assistant\n", ) return (trainer_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's verify masking the instruction part is done! Let's print the 100th row again. """) return @app.cell def _(tokenizer, trainer_1): tokenizer.decode(trainer_1.train_dataset[100]["input_ids"]) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now let's print the masked out example - you should see only the answer is present: """) return @app.cell def _(tokenizer, trainer_1): tokenizer.decode( [ tokenizer.pad_token_id if x == -100 else x for x in trainer_1.train_dataset[100]["labels"] ] ).replace(tokenizer.pad_token, " ") return @app.cell def _(trainer_1): # Compilation can take 2-3 minutes of time, so please be patient :) trainer_1.train() return @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's check if the model has learnt to follow the custom format: """) return @app.cell def _(model_1, tokenizer): messages = [{"role": "user", "content": "Continue the sequence: 1, 1, 2, 3, 5, 8,"}] text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) from transformers import TextStreamer _ = model_1.generate( **tokenizer(text, return_tensors="pt").to("cuda"), max_new_tokens=1000, # Increase for longer outputs! temperature=0.7, # For non thinking top_p=0.8, top_k=20, streamer=TextStreamer(tokenizer, skip_prompt=True), ) return @app.cell def _(dataset_2, torch): del dataset_2 torch.cuda.empty_cache() import gc gc.collect() return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1, tokenizer): # Merge to 16bit if False: model_1.save_pretrained_merged( "qwen_finetune_16bit", tokenizer, save_method="merged_16bit" ) if False: # Merge to 4bit model_1.push_to_hub_merged( "HF_USERNAME/qwen_finetune_16bit", tokenizer, save_method="merged_16bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained_merged( "qwen_finetune_4bit", tokenizer, save_method="merged_4bit" ) # Just LoRA adapters if False: model_1.push_to_hub_merged( "HF_USERNAME/qwen_finetune_4bit", tokenizer, save_method="merged_4bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained("qwen_lora") tokenizer.save_pretrained("qwen_lora") if False: model_1.push_to_hub("HF_USERNAME/qwen_lora", token="YOUR_HF_TOKEN") tokenizer.push_to_hub("HF_USERNAME/qwen_lora", token="YOUR_HF_TOKEN") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### GGUF / llama.cpp Conversion To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF. Some supported quant methods (full list on our [docs page](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf)): * `q8_0` - Fast conversion. High resource use, but generally acceptable. * `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K. * `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K. [**NEW**] To finetune and auto export to Ollama, try our [Ollama notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) """) return @app.cell def _(model_1, tokenizer): # Save to 8bit Q8_0 if False: # Remember to go to https://huggingface.co/settings/tokens for a token! # And change hf to your username! model_1.save_pretrained_gguf("qwen_finetune", tokenizer) if False: # Save to 16bit GGUF model_1.push_to_hub_gguf( "HF_USERNAME/qwen_finetune", tokenizer, token="YOUR_HF_TOKEN" ) if False: model_1.save_pretrained_gguf( "qwen_finetune", tokenizer, quantization_method="f16" ) # Save to q4_k_m GGUF if False: model_1.push_to_hub_gguf( "HF_USERNAME/qwen_finetune", tokenizer, quantization_method="f16", token="YOUR_HF_TOKEN", ) if False: # Save to multiple GGUF options - much faster if you want multiple! model_1.save_pretrained_gguf( "qwen_finetune", tokenizer, quantization_method="q4_k_m" ) if False: model_1.push_to_hub_gguf( "HF_USERNAME/qwen_finetune", tokenizer, quantization_method="q4_k_m", token="YOUR_HF_TOKEN", ) # Change hf to your username! if False: model_1.push_to_hub_gguf( "HF_USERNAME/qwen_finetune", tokenizer, quantization_method=["q4_k_m", "q8_0", "q5_k_m"], token="YOUR_HF_TOKEN", ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now, use the `qwen_finetune.Q8_0.gguf` file or `qwen_finetune.Q4_K_M.gguf` file in llama.cpp. And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Train your own reasoning model - Llama GRPO notebook [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb) 2. Saving finetunes to Ollama. [Free notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) 3. Llama 3.2 Vision finetuning - Radiography use case. [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb) 4. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://unsloth.ai/docs/get-started/unsloth-notebooks)!

Join Discord if you need help + ⭐️ Star us on Github ⭐️

This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). """) return if __name__ == "__main__": app.run()