# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "timm", # "torchao>=0.16.0", # "torchcodec", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell def _(): # Only for Gemma 3N return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth `FastModel` supports loading nearly any model now! This includes Vision and Text models! """) return @app.cell def _(): from unsloth import FastModel import torch from huggingface_hub import snapshot_download fourbit_models = [ # 4bit dynamic quants for superior accuracy and low memory use "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit", "unsloth/gemma-3n-E2B-it-unsloth-bnb-4bit", # Pretrained models "unsloth/gemma-3n-E4B-unsloth-bnb-4bit", "unsloth/gemma-3n-E2B-unsloth-bnb-4bit", # Other Gemma 3 quants "unsloth/gemma-3-1b-it-unsloth-bnb-4bit", "unsloth/gemma-3-4b-it-unsloth-bnb-4bit", "unsloth/gemma-3-12b-it-unsloth-bnb-4bit", "unsloth/gemma-3-27b-it-unsloth-bnb-4bit", ] # More models at https://huggingface.co/unsloth model, processor = FastModel.from_pretrained( model_name="unsloth/gemma-3n-E4B-it", # YOUR MODEL YOU USED FOR TRAINING dtype=None, # None for auto detection max_seq_length=1024, # Choose any for long context! load_in_4bit=True, # 4 bit quantization to reduce memory full_finetuning=False, # [NEW!] We have full finetuning now! # token = "YOUR_HF_TOKEN", # HF Token for gated models ) return FastModel, model, processor, torch @app.cell(hide_code=True) def _(mo): mo.md(r""" # Gemma 3N can process Text, Vision and Audio! Let's first experience how Gemma 3N can handle multimodal inputs. We use Gemma 3N's recommended settings of `temperature = 1.0, top_p = 0.95, top_k = 64` but for this example we use `do_sample=False` for ASR. """) return @app.cell def _(model, processor): from transformers import TextStreamer # Helper function for inference def do_gemma_3n_inference(messages, max_new_tokens=128): _ = model.generate( **processor.apply_chat_template( messages, add_generation_prompt=True, # Must add for generation tokenize=True, return_dict=True, return_tensors="pt", ).to("cuda"), max_new_tokens=max_new_tokens, # Increase for longer outputs! do_sample=False, streamer=TextStreamer(processor, skip_prompt=True), ) return TextStreamer, do_gemma_3n_inference @app.cell(hide_code=True) def _(mo): mo.md(r"""

Let's Evaluate Gemma 3N Baseline Performance on German Transcription

""") return @app.cell def _(): from datasets import load_dataset, Audio, concatenate_datasets dataset = load_dataset("kadirnar/Emilia-DE-B000000", split="train") test_audio = dataset[7546] dataset = dataset.select(range(3000)) dataset = dataset.cast_column("audio", Audio(sampling_rate=16000)) return Audio, dataset, test_audio @app.cell def _(Audio, test_audio): from IPython.display import display print(test_audio["text"]) Audio(test_audio["audio"]["array"], rate=test_audio["audio"]["sampling_rate"]) return @app.cell def _(do_gemma_3n_inference, test_audio): messages = [ { "role": "system", "content": [ { "type": "text", "text": "You are an assistant that transcribes speech accurately.", } ], }, { "role": "user", "content": [ {"type": "audio", "audio": test_audio["audio"]["array"]}, {"type": "text", "text": "Please transcribe this audio."}, ], }, ] do_gemma_3n_inference(messages, max_new_tokens=256) return @app.cell(hide_code=True) def _(mo): mo.md(r"""

Baseline Model Performance: 24.32% Word Error Rate (WER) for this sample !

""") return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Let's finetune Gemma 3N! You can finetune the vision and text and audio parts """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters so we only need to update a small amount of parameters! """) return @app.cell def _(FastModel, model): model_1 = FastModel.get_peft_model( model, finetune_vision_layers=False, # False if not finetuning vision layers finetune_language_layers=True, # False if not finetuning language layers finetune_attention_modules=True, # False if not finetuning attention layers finetune_mlp_modules=True, # False if not finetuning MLP layers r=8, # The larger, the higher the accuracy, but might overfit lora_alpha=16, # Recommended alpha == r at least lora_dropout=0, bias="none", random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "post", "linear_start", "linear_end", "embedding_projection", ], ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We adapt the `kadirnar/Emilia-DE-B000000` dataset for our German ASR task using Gemma 3N multi-modal chat format. Each audio-text pair is structured into a conversation with `system`, `user`, and `assistant` roles. The processor then converts this into the final training format: ``` system You are an assistant that transcribes speech accurately. user

Let's Evaluate Gemma 3N Baseline Performance on German Transcription

Baseline Model Performance: 24.32% Word Error Rate (WER) for this sample !

With only 3,000 German speech samples, we reduced the Word Error Rate (WER) from 24.32% to 16.22%. This represents a significant 33.31% relative error rate reduction !