# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell def _(): from unsloth import FastLanguageModel import torch max_seq_length = 4096 # Choose any! We auto support RoPE Scaling internally! dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False. # 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/mistral-7b-bnb-4bit", "unsloth/mistral-7b-instruct-v0.2-bnb-4bit", "unsloth/llama-2-7b-bnb-4bit", "unsloth/gemma-7b-bnb-4bit", "unsloth/gemma-7b-it-bnb-4bit", # Instruct version of Gemma 7b "unsloth/gemma-2b-bnb-4bit", "unsloth/gemma-2b-it-bnb-4bit", # Instruct version of Gemma 2b "unsloth/llama-3-8b-bnb-4bit", # [NEW] 15 Trillion token Llama-3 ] # More models at https://huggingface.co/unsloth model, tokenizer = FastLanguageModel.from_pretrained( model_name="unsloth/llama-3-8b-bnb-4bit", # YOUR MODEL YOU USED FOR TRAINING max_seq_length=max_seq_length, # Choose any! We auto support RoPE Scaling internally! dtype=dtype, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit=load_in_4bit, # Use 4bit quantization to reduce memory usage. Can be False. # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return ( FastLanguageModel, dtype, load_in_4bit, max_seq_length, model, tokenizer, ) @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters so we only need to update 1 to 10% of all parameters! """) return @app.cell def _(FastLanguageModel, model): model_1 = FastLanguageModel.get_peft_model( model, r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], lora_alpha=16, lora_dropout=0, # Supports any, but = 0 is optimized bias="none", # Supports any, but = "none" is optimized use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We now use a special ORPO style dataset from [recipe-research](https://huggingface.co/datasets/reciperesearch/dolphin-sft-v0.1-preference). You need at least 3 columns: * Instruction * Accepted * Rejected For example: * Instruction: "What is 2+2?" * Accepted: "The answer is 4" * Rejected: "The answer is 5" The goal of ORPO is to penalize the "rejected" samples, and increase the likelihood of "accepted" samples. [recipe-research](https://huggingface.co/datasets/reciperesearch/dolphin-sft-v0.1-preference) essentially used Mistral to generate the "rejected" responses, and used GPT-4 to generated the "accepted" responses. """) return @app.cell def _(tokenizer): # The data must be formatted with appropriate prompt template first. # See details here: https://github.com/huggingface/trl/blob/main/examples/scripts/orpo.py alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. ### Instruction: {} ### Input: {} ### Response: {}""" EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN def format_prompt(sample): instruction = sample["instruction"] input = sample["input"] accepted = sample["accepted"] rejected = sample["rejected"] # ORPOTrainer expects prompt/chosen/rejected keys # See: https://huggingface.co/docs/trl/main/en/orpo_trainer sample["prompt"] = alpaca_prompt.format(instruction, input, "") sample["chosen"] = accepted + EOS_TOKEN sample["rejected"] = rejected + EOS_TOKEN return sample from datasets import load_dataset dataset = load_dataset("reciperesearch/dolphin-sft-v0.1-preference")["train"] dataset = dataset.map( format_prompt, ) return alpaca_prompt, dataset @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's print out some examples to see how the dataset should look like """) return @app.cell def _(dataset): import pprint row = dataset[1] print("INSTRUCTION: " + "=" * 50) pprint.pprint(row["prompt"]) print("ACCEPTED: " + "=" * 50) pprint.pprint(row["chosen"]) print("REJECTED: " + "=" * 50) pprint.pprint(row["rejected"]) return @app.cell def _(): # Enable reward modelling stats from unsloth import PatchDPOTrainer PatchDPOTrainer() return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. """) return @app.cell def _(dataset, max_seq_length, model_1, tokenizer): from trl import ORPOConfig, ORPOTrainer orpo_trainer = ORPOTrainer( model=model_1, train_dataset=dataset, tokenizer=tokenizer, args=ORPOConfig( max_length=max_seq_length, max_prompt_length=max_seq_length // 2, max_completion_length=max_seq_length // 2, per_device_train_batch_size=2, gradient_accumulation_steps=4, beta=0.1, logging_steps=1, optim="adamw_8bit", lr_scheduler_type="linear", max_steps=30, # Change to num_train_epochs = 1 for full training runs output_dir="outputs", report_to="none", # Use TrackIO/WandB etc ), ) return (orpo_trainer,) @app.cell def _(orpo_trainer): orpo_trainer.train() return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inference Let's run the model! You can change the instruction and input - leave the output blank! """) return @app.cell def _(FastLanguageModel, alpaca_prompt, model_1, tokenizer): # alpaca_prompt = Copied from above FastLanguageModel.for_inference(model_1) # Enable native 2x faster inference inputs = tokenizer( [ alpaca_prompt.format( "Continue the fibonacci sequence.", "1, 1, 2, 3, 5, 8", "" ) ], return_tensors="pt", ).to("cuda") outputs = model_1.generate(**inputs, max_new_tokens=64, use_cache=True) tokenizer.batch_decode( outputs ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" You can also use a `TextStreamer` for continuous inference - so you can see the generation token by token, instead of waiting the whole time! """) return @app.cell def _(FastLanguageModel, alpaca_prompt, model_1, tokenizer): # alpaca_prompt = Copied from above FastLanguageModel.for_inference(model_1) # Enable native 2x faster inference inputs_1 = tokenizer( [ alpaca_prompt.format( "Continue the fibonacci sequence.", "1, 1, 2, 3, 5, 8", "" ) ], return_tensors="pt", ).to("cuda") from transformers import TextStreamer text_streamer = TextStreamer(tokenizer) _ = model_1.generate( **inputs_1, streamer=text_streamer, max_new_tokens=128 ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving, loading finetuned models To save the final model as LoRA adapters, either use Hugging Face's `push_to_hub` for an online save or `save_pretrained` for a local save. **[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down! """) return @app.cell def _(model_1, tokenizer): model_1.save_pretrained("llama_lora") tokenizer.save_pretrained("llama_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`: """) return @app.cell def _(alpaca_prompt, dtype, load_in_4bit, max_seq_length, model_1, tokenizer): if False: from unsloth import FastLanguageModel as _FastLanguageModel _model, _tokenizer = _FastLanguageModel.from_pretrained( model_name="llama_lora", # YOUR MODEL YOU USED FOR TRAINING max_seq_length=max_seq_length, # Choose any! We auto support RoPE Scaling internally! dtype=dtype, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ load_in_4bit=load_in_4bit, # Use 4bit quantization to reduce memory usage. Can be False. ) _FastLanguageModel.for_inference(_model) inputs_2 = tokenizer( [alpaca_prompt.format("What is a famous tall tower in Paris?", "", "")], return_tensors="pt", ).to("cuda") # alpaca_prompt = You MUST copy from above! outputs_1 = model_1.generate(**inputs_2, max_new_tokens=64, use_cache=True) tokenizer.batch_decode( outputs_1 ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" You can also use Hugging Face's `AutoPeftModelForCausalLM`. Only use this if you do not have `unsloth` installed. It can be hopelessly slow, since `4bit` model downloading is not supported, and Unsloth's **inference is 2x faster**. """) return @app.cell def _(load_in_4bit): if False: from peft import AutoPeftModelForCausalLM as _AutoPeftModelForCausalLM from transformers import AutoTokenizer as _AutoTokenizer _model = _AutoPeftModelForCausalLM.from_pretrained( "llama_lora", load_in_4bit=load_in_4bit ) _tokenizer = _AutoTokenizer.from_pretrained("llama_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1, tokenizer): # Merge to 16bit if False: model_1.save_pretrained_merged( "llama_finetune_16bit", tokenizer, save_method="merged_16bit" ) if False: # Merge to 4bit model_1.push_to_hub_merged( "HF_USERNAME/llama_finetune_16bit", tokenizer, save_method="merged_16bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained_merged( "llama_finetune_4bit", tokenizer, save_method="merged_4bit" ) # Just LoRA adapters if False: model_1.push_to_hub_merged( "HF_USERNAME/llama_finetune_4bit", tokenizer, save_method="merged_4bit", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained("llama_lora") tokenizer.save_pretrained("llama_lora") if False: model_1.push_to_hub("HF_USERNAME/llama_lora", token="YOUR_HF_TOKEN") tokenizer.push_to_hub("HF_USERNAME/llama_lora", token="YOUR_HF_TOKEN") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### GGUF / llama.cpp Conversion To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF. Some supported quant methods (full list on our [docs page](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf)): * `q8_0` - Fast conversion. High resource use, but generally acceptable. * `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K. * `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K. """) return @app.cell def _(model_1, tokenizer): # Save to 8bit Q8_0 if False: model_1.save_pretrained_gguf("llama_finetune", tokenizer) if False: # Save to 16bit GGUF model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, token="YOUR_HF_TOKEN" ) if False: model_1.save_pretrained_gguf( "llama_finetune", tokenizer, quantization_method="f16" ) # Save to q4_k_m GGUF if False: model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, quantization_method="f16", token="YOUR_HF_TOKEN", ) if False: model_1.save_pretrained_gguf( "llama_finetune", tokenizer, quantization_method="q4_k_m" ) if False: model_1.push_to_hub_gguf( "HF_USERNAME/llama_finetune", tokenizer, quantization_method="q4_k_m", token="YOUR_HF_TOKEN", ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()