# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "hf_transfer", # "huggingface_hub>=0.34.0", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "torchcodec", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook on your A100 molab Pro instance, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Launch sglang inference for unsloth/gemma-3n-E2B-it (https://huggingface.co/unsloth/gemma-3n-E2B-it) """) return @app.cell def _(): import subprocess import subprocess # Load and run the model using sglang #! nohup python -m sglang.launch_server --model-path unsloth/gemma-3n-E2B-it --attention-backend fa3 --port 8000 > sglang.log & subprocess.call( "nohup python -m sglang.launch_server --model-path unsloth/gemma-3n-E2B-it --attention-backend fa3 --port 8000 > sglang.log &", shell=True, ) # tail vllm logs. Check server has been started correctly #! while ! grep -q "The server is fired up and ready to roll" sglang.log; do tail -n 1 sglang.log; sleep 5; done subprocess.call( "while ! grep -q 'The server is fired up and ready to roll' sglang.log; do tail -n 1 sglang.log; sleep 5; done", shell=True, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Image helper functions """) return @app.cell def _(): from PIL.ImageFile import ImageFile from PIL import Image import numpy as np import io import base64 import requests from io import BytesIO def load_image_from_url(url): response = requests.get(url) img = Image.open(BytesIO(response.content)) return img def process_image(image: ImageFile) -> str: """Process image for sglang gemma3n and return base64 string.""" assert isinstance(image, ImageFile), "please pass an image object" # Resize the image resized_image = image.resize((384, 384)) # Convert to numpy array and transpose image_array = np.array(resized_image) array_reordered = np.transpose(image_array, (1, 0, 2)) # Convert back to PIL Image processed_image = Image.fromarray(array_reordered) # Convert to base64 string image_bytes = io.BytesIO() processed_image.save(image_bytes, format=image.format) base64_image = base64.b64encode(image_bytes.getvalue()).decode("utf-8") # Return data URL string format_name = image.format.lower() if image.format else "png" return f"data:image/{format_name};base64,{base64_image}" return load_image_from_url, process_image, requests @app.cell(hide_code=True) def _(mo): mo.md(r""" ## Gemma3n Inference using sglang (source model: https://huggingface.co/unsloth/gemma-3n-E2B-it) """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ## Inference 1 Image source file "https://raw.githubusercontent.com/sgl-project/sglang/refs/heads/main/test/lang/example_image.png" """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" load image from url source """) return @app.cell def _(load_image_from_url): from IPython.display import display image = load_image_from_url( "https://raw.githubusercontent.com/sgl-project/sglang/refs/heads/main/test/lang/example_image.png" ) display(image) return display, image @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's run the model! """) return @app.cell def _(image, process_image, requests): from sglang.utils import wait_for_server, print_highlight, terminate_process processed_image = process_image(image) url = f"http://localhost:8000/v1/chat/completions" processed_image = process_image(image) data = { "model": "unsloth/gemma-3n-E2B-it", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "What’s in this image?"}, {"type": "image_url", "image_url": {"url": processed_image}}, ], } ], "max_tokens": 300, } response = requests.post(url, json=data) print_highlight(response.text) return (print_highlight,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ## Inference 2 Image source file "https://i.ibb.co/1tw5whfz/ocr.png" """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" load image from url source """) return @app.cell def _(display, load_image_from_url): image_1 = load_image_from_url("https://i.ibb.co/1tw5whfz/ocr.png") display(image_1) return (image_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's run the model! """) return @app.cell def _(image_1, print_highlight, process_image, requests): url_1 = f"http://localhost:8000/v1/chat/completions" processed_image_1 = process_image(image_1) data_1 = { "model": "unsloth/gemma-3n-E2B-it", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "Read the text in the image"}, {"type": "image_url", "image_url": {"url": processed_image_1}}, ], } ], "max_tokens": 300, } response_1 = requests.post(url_1, json=data_1) print_highlight(response_1.text) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Looking to use Unsloth locally? Read our [Installation Guide](https://unsloth.ai/docs/get-started/install) for details on installing Unsloth on Windows, Docker, AMD, Intel GPUs. 2. Learn how to do Reinforcement Learning with our [RL Guide and notebooks](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide). 3. Read our guides and notebooks for [Text-to-speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning) and [vision](https://unsloth.ai/docs/basics/vision-fine-tuning) model support. 4. Explore our [LLM Tutorials Directory](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms) to find dedicated guides for each model. 5. Need help with Inference? Read our [Inference & Deployment page](https://unsloth.ai/docs/basics/inference-and-deployment) for details on using vLLM, llama.cpp, Ollama etc.

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()