# /// script # requires-python = ">=3.10,<3.14" # dependencies = [ # "accelerate", # "addict", # "bitsandbytes>=0.43.0", # "datasets==4.3.0", # "easydict", # "einops", # "hf_transfer", # "huggingface_hub>=0.34.0", # "jiwer", # "marimo", # "peft", # "protobuf", # "sentencepiece", # "torchao>=0.16.0", # "transformers==4.56.2", # "triton>=3.2.0", # "trl==0.22.2", # "unsloth @ git+https://github.com/unslothai/unsloth", # "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo", # ] # # [tool.uv] # no-build-package = [ # "bitsandbytes", # "triton", # "vllm", # "xformers", # ] # /// import marimo __generated_with = "0.23.8" app = marimo.App() @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" To run this notebook on your A100 molab Pro instance, hit the **▶ Run all** button in the bottom-right corner - or use `Ctrl/Cmd + Shift + R`.

Join Discord if you need help + ⭐ Star us on Github ⭐

To install Unsloth on your local device, follow [our guide](https://unsloth.ai/docs/get-started/install). This notebook is licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme). You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & how to save it """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### News """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Introducing **Unsloth Studio** - a new open source, no-code web UI to train and run LLMs. [Blog](https://unsloth.ai/docs/new/studio) • [Notebook](https://github.com/unslothai/unsloth/blob/main/studio/Unsloth_Studio_Colab.ipynb)

_{Train models — no code needed}

_{Run GGUF models on Mac, Windows & Linux}

Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. [Blog](https://unsloth.ai/docs/new/faster-moe) Ultra Long-Context Reinforcement Learning is here with 7x more context windows! [Blog](https://unsloth.ai/docs/new/grpo-long-context) New in Reinforcement Learning: [FP8 RL](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Vision RL](https://unsloth.ai/docs/new/vision-reinforcement-learning-vlm-rl) • [Standby](https://unsloth.ai/docs/basics/memory-efficient-rl) • [gpt-oss RL](https://unsloth.ai/docs/new/gpt-oss-reinforcement-learning) Visit our docs for all our [model uploads](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unsloth """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's prepare the OCR model to our local first """) return @app.cell def _(): from huggingface_hub import snapshot_download snapshot_download("unsloth/DeepSeek-OCR", local_dir="deepseek_ocr") return @app.cell def _(): from unsloth import FastVisionModel # FastLanguageModel for LLMs import torch from transformers import AutoModel import os os.environ["UNSLOTH_WARN_UNINITIALIZED"] = "0" # 4bit pre quantized models we support for 4x faster downloading + no OOMs. fourbit_models = [ "unsloth/Qwen3-VL-8B-Instruct-bnb-4bit", # Qwen 3 vision support "unsloth/Qwen3-VL-8B-Thinking-bnb-4bit", "unsloth/Qwen3-VL-32B-Instruct-bnb-4bit", "unsloth/Qwen3-VL-32B-Thinking-bnb-4bit", ] # More models at https://huggingface.co/unsloth model, tokenizer = FastVisionModel.from_pretrained( "./deepseek_ocr", load_in_4bit=False, # Use 4bit to reduce memory use. False for 16bit LoRA. auto_model=AutoModel, trust_remote_code=True, unsloth_force_compile=True, use_gradient_checkpointing="unsloth", # True or "unsloth" for long context ) return AutoModel, FastVisionModel, model, os, tokenizer, torch @app.cell def _(os): # @title Create evaluation functions import json from typing import Dict import numpy as np from jiwer import cer from tqdm import tqdm from datasets import load_dataset def calculate_cer(ref: str, hyp: str) -> float: """Helper to calculate CER and convert to percentage.""" return cer(ref, hyp) * 100 def evaluate_model( model, tokenizer, dataset, num_samples: int = 100, base_size: int = 1024, image_size: int = 640, crop_mode: bool = True, verbose: bool = True, ): """ Runs the model over a subset of the dataset to see how it performs. It'll calculate CER stats and save all the predictions. """ results = { "cer_scores": [], "predictions": [], "references": [], "sample_indices": [], } num_samples = min(num_samples, len(dataset)) indices = np.linspace(0, len(dataset) - 1, num_samples, dtype=int) iterator = tqdm(indices, desc="Evaluating") if verbose else indices for idx in iterator: sample = dataset[int(idx)] temp_image_path = f"temp_eval_image_{idx}.jpg" sample["image_path"].save(temp_image_path) prediction = "" reference = sample["text"].strip() try: prediction = model.infer( tokenizer, prompt="\nFree OCR. ", image_file=temp_image_path, output_path="temp_output", base_size=base_size, image_size=image_size, crop_mode=crop_mode, eval_mode=True, save_results=False, test_compress=False, ) prediction = prediction.strip() cer_score = calculate_cer(reference, prediction) results["cer_scores"].append(cer_score) results["predictions"].append(prediction) results["references"].append(reference) results["sample_indices"].append(int(idx)) except Exception as e: print(f"\nError processing sample {idx}: {e}") print(f"Reference was: {reference}") continue # make sure we don't try to sample more than we have finally: if os.path.exists(temp_image_path): os.remove( temp_image_path ) # Grab evenly spaced samples from the dataset if results["cer_scores"]: results["mean_cer"] = np.mean(results["cer_scores"]) results["median_cer"] = np.median( results["cer_scores"] ) # Use tqdm for a progress bar if verbose results["std_cer"] = np.std(results["cer_scores"]) results["min_cer"] = np.min(results["cer_scores"]) results["max_cer"] = np.max(results["cer_scores"]) else: print("Warning: No samples were successfully processed.") results[ "mean_cer" ] = ( -1.0 ) # The model.infer method needs a file path, so we save a temp image results["num_samples"] = len(results["cer_scores"]) return results def print_evaluation_summary(results: Dict, title: str = "Evaluation Results"): """Prints a nice summary of the stats to the console.""" print("\n" + "=" * 60) print(f"{title}") print("=" * 60) # Run the actual inference print(f"Number of samples: {results['num_samples']}") print(f"Mean CER: {results['mean_cer']:.2f}%") print(f"Median CER: {results['median_cer']:.2f}%") print(f"Std Dev: {results['std_cer']:.2f}%") print(f"Min CER: {results['min_cer']:.2f}%") print(f"Max CER: {results['max_cer']:.2f}%") print("=" * 60) sorted_indices = np.argsort(results["cer_scores"]) print("\n Best Predictions (Lowest CER):") for i in range(min(3, len(sorted_indices))): idx = sorted_indices[i] print( f"\nSample {results['sample_indices'][idx]} (CER: {results['cer_scores'][idx]:.2f}%)" ) print(f"Reference: {results['references'][idx][:100]}...") print(f"Prediction: {results['predictions'][idx][:100]}...") print("\n Worst Predictions (Highest CER):") for i in range(min(3, len(sorted_indices))): # Calculate CER idx = sorted_indices[-(i + 1)] print( f"\nSample {results['sample_indices'][idx]} (CER: {results['cer_scores'][idx]:.2f}%)" ) print(f"Reference: {results['references'][idx][:100]}...") print(f"Prediction: {results['predictions'][idx][:100]}...") def save_evaluation_results(results: Dict, filepath: str): """Save full results dictionary to a JSON file.""" with open(filepath, "w", encoding="utf-8") as f: json.dump( results, f, ensure_ascii=False, indent=2 ) # Don't let one bad sample crash the whole evaluation print( f"\n✅ Results saved to {filepath}" ) return ( Dict, evaluate_model, load_dataset, print_evaluation_summary, save_evaluation_results, ) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Let's Evaluate Deepseek-OCR Baseline Performance on Persian Transcription """) return @app.cell def _( evaluate_model, load_dataset, model, print_evaluation_summary, save_evaluation_results, tokenizer, ): print("Loading evaluation dataset...") eval_dataset = load_dataset("hezarai/parsynth-ocr-200k", split="test") print("\n Running Baseline Evaluation...") baseline_results = evaluate_model( model=model, tokenizer=tokenizer, dataset=eval_dataset, num_samples=200, base_size=1024, image_size=640, crop_mode=True, verbose=True, ) print_evaluation_summary(baseline_results, "Baseline Model Performance") save_evaluation_results(baseline_results, "baseline_evaluation.json") return baseline_results, eval_dataset @app.cell(hide_code=True) def _(mo): mo.md(r"""

Mean Baseline Model Performance: 149.07% Character Error Rate (CER) for this eval set !

""") return @app.cell(hide_code=True) def _(mo): mo.md(r""" # Let's finetune Deepseek-OCR ! """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" We now add LoRA adapters for parameter efficient finetuning - this allows us to only efficiently train 1% of all parameters. **[NEW]** We also support finetuning ONLY the vision part of the model, or ONLY the language part. Or you can select both! You can also select to finetune the attention or the MLP layers! """) return @app.cell def _(FastVisionModel, model): model_1 = FastVisionModel.get_peft_model( model, target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], r=16, # The larger, the higher the accuracy, but might overfit lora_alpha=16, # Recommended alpha == r at least lora_dropout=0, bias="none", random_state=3407, use_rslora=False, # We support rank stabilized LoRA loftq_config=None, # And LoftQ ) return (model_1,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Data Prep We'll be using a dataset for Persian OCR. The goal is to convert these images into a computer readable form - ie text. This can be very useful for digitizing Persian text. You can access the dataset [here](https://huggingface.co/datasets/hezarai/parsynth-ocr-200k). """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's take an overview look at the dataset. We shall see what the 3rd image is, and what caption it had. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" To format the dataset, all vision finetuning tasks should be formatted as follows: ```python [ { "role": "<|User|>", "content": "\", "images": [] }, { "role": "<|Assistant|>", "content": "\" }, ] ``` """) return @app.cell def _(load_dataset): instruction = "\nFree OCR. " def convert_to_conversation(sample): """Convert dataset sample to conversation format""" conversation = [ {"role": "<|User|>", "content": instruction, "images": [sample["image"]]}, {"role": "<|Assistant|>", "content": sample["text"]}, ] return {"messages": conversation} dataset = load_dataset("hezarai/parsynth-ocr-200k", split="train[:1000]") # Load dataset dataset = dataset.rename_column("image_path", "image") return convert_to_conversation, dataset @app.cell(hide_code=True) def _(mo): mo.md(r""" Let's convert the dataset into the "correct" format for finetuning: """) return @app.cell def _(convert_to_conversation, dataset): converted_dataset = [convert_to_conversation(sample) for sample in dataset] return (converted_dataset,) @app.cell(hide_code=True) def _(mo): mo.md(r""" We look at how the conversations are structured for the first example: """) return @app.cell def _(converted_dataset): converted_dataset[0] return @app.cell def _(Dict, torch): # @title Create datacollator import math from dataclasses import dataclass from typing import List, Any, Tuple from PIL import Image, ImageOps from torch.nn.utils.rnn import pad_sequence import io from deepseek_ocr.modeling_deepseekocr import ( format_messages, text_encode, BasicImageTransform, dynamic_preprocess, ) @dataclass class DeepSeekOCRDataCollator: """ Data collator that handles image preprocessing and tokenization at batch time. Args: tokenizer: Tokenizer instance model: Model instance (used to get dtype) image_size: Size for image patches (default: 640) base_size: Size for global view (default: 1024) crop_mode: Whether to use dynamic cropping for large images train_on_responses_only: If True, only train on assistant responses (mask user prompts) """ tokenizer: Any model: Any image_size: int = 640 base_size: int = 1024 crop_mode: bool = True image_token_id: int = 128815 train_on_responses_only: bool = True def __init__( self, tokenizer, model, image_size: int = 640, base_size: int = 1024, crop_mode: bool = True, train_on_responses_only: bool = True, ): self.tokenizer = tokenizer self.model = model self.image_size = image_size self.base_size = base_size self.crop_mode = crop_mode self.image_token_id = 128815 self.dtype = model.dtype self.train_on_responses_only = train_on_responses_only self.image_transform = BasicImageTransform( mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True ) self.patch_size = 16 self.downsample_ratio = 4 if ( hasattr(tokenizer, "bos_token_id") and tokenizer.bos_token_id is not None ): self.bos_id = tokenizer.bos_token_id else: self.bos_id = 0 print( f"Warning: tokenizer has no bos_token_id, using default: {self.bos_id}" ) def deserialize_image(self, image_data) -> Image.Image: """Convert image data (bytes dict or PIL Image) to PIL Image in RGB mode""" if isinstance(image_data, Image.Image): return image_data.convert("RGB") elif isinstance(image_data, dict) and "bytes" in image_data: image_bytes = image_data["bytes"] # Get dtype from model image = Image.open(io.BytesIO(image_bytes)) return image.convert("RGB") else: raise ValueError(f"Unsupported image format: {type(image_data)}") def calculate_image_token_count( self, image: Image.Image, crop_ratio: Tuple[int, int] ) -> int: """Calculate the number of tokens this image will generate""" num_queries = math.ceil( self.image_size // self.patch_size / self.downsample_ratio ) num_queries_base = math.ceil( self.base_size // self.patch_size / self.downsample_ratio ) width_crop_num, height_crop_num = crop_ratio if self.crop_mode: # Get BOS token ID from tokenizer img_tokens = num_queries_base * num_queries_base + 1 if width_crop_num > 1 or height_crop_num > 1: img_tokens = img_tokens + (num_queries * width_crop_num + 1) * ( num_queries * height_crop_num ) else: img_tokens = num_queries * num_queries + 1 return img_tokens def process_image( self, image: Image.Image ) -> Tuple[List, List, List, List, Tuple[int, int]]: """ Process a single image based on crop_mode and size thresholds Returns: Tuple of (images_list, images_crop_list, images_spatial_crop, tokenized_image, crop_ratio) """ images_list = [] images_crop_list = [] images_spatial_crop = [] if self.crop_mode: if image.size[0] <= 640 and image.size[1] <= 640: crop_ratio = (1, 1) images_crop_raw = [] else: images_crop_raw, crop_ratio = dynamic_preprocess( image, min_num=2, max_num=9, image_size=self.image_size, use_thumbnail=False, ) global_view = ImageOps.pad( image, (self.base_size, self.base_size), color=tuple((int(x * 255) for x in self.image_transform.mean)), ) images_list.append(self.image_transform(global_view).to(self.dtype)) width_crop_num, height_crop_num = crop_ratio images_spatial_crop.append([width_crop_num, height_crop_num]) if width_crop_num > 1 or height_crop_num > 1: for crop_img in images_crop_raw: images_crop_list.append( self.image_transform(crop_img).to(self.dtype) ) num_queries = math.ceil( self.image_size // self.patch_size / self.downsample_ratio ) num_queries_base = math.ceil( self.base_size // self.patch_size / self.downsample_ratio ) tokenized_image = ( [self.image_token_id] * num_queries_base + [self.image_token_id] ) * num_queries_base tokenized_image = tokenized_image + [self.image_token_id] if width_crop_num > 1 or height_crop_num > 1: tokenized_image = tokenized_image + ( [self.image_token_id] * (num_queries * width_crop_num) + [self.image_token_id] ) * (num_queries * height_crop_num) else: crop_ratio = (1, 1) images_spatial_crop.append([1, 1]) if self.base_size <= 640: resized_image = image.resize( (self.base_size, self.base_size), Image.LANCZOS ) images_list.append( self.image_transform(resized_image).to(self.dtype) ) else: global_view = ImageOps.pad( image, (self.base_size, self.base_size), color=tuple((int(x * 255) for x in self.image_transform.mean)), ) images_list.append(self.image_transform(global_view).to(self.dtype)) num_queries = math.ceil( self.base_size // self.patch_size / self.downsample_ratio ) # Determine crop ratio based on image size tokenized_image = ( [self.image_token_id] * num_queries + [self.image_token_id] ) * num_queries tokenized_image = tokenized_image + [self.image_token_id] return ( images_list, images_crop_list, images_spatial_crop, tokenized_image, crop_ratio, ) def process_single_sample(self, messages: List[Dict]) -> Dict[str, Any]: """ Process a single conversation into model inputs. This version builds the token sequence in a single pass, accurately calculating the prompt/response split point. # Process global view with padding """ images = [] for message in messages: if "images" in message and message["images"]: for img_data in message["images"]: if img_data is not None: pil_image = self.deserialize_image(img_data) images.append(pil_image) if not images: raise ValueError( "No images found in sample. Please ensure all samples contain images." ) # Process local views (crops) if applicable tokenized_str = [] images_seq_mask = [] images_list, images_crop_list, images_spatial_crop = ([], [], []) prompt_token_count = -1 # Index to start training assistant_started = False image_idx = 0 tokenized_str.append(self.bos_id) # Calculate image tokens images_seq_mask.append(False) for message in messages: role = message["role"] content = message["content"] if role == "<|Assistant|>": if not assistant_started: prompt_token_count = len(tokenized_str) # Index to start training assistant_started = True content = f"{content.strip()} {self.tokenizer.eos_token}" text_splits = content.split("") for i, text_sep in enumerate(text_splits): # crop_mode = False tokenized_sep = text_encode( self.tokenizer, text_sep, bos=False, eos=False ) tokenized_str.extend(tokenized_sep) images_seq_mask.extend([False] * len(tokenized_sep)) if ( i < len(text_splits) - 1 ): # For smaller base sizes, resize; for larger, pad if image_idx >= len(images): raise ValueError( f"Data mismatch: Found '' token but no corresponding image." ) image = images[image_idx] img_list, crop_list, spatial_crop, tok_img, _ = ( self.process_image(image) ) images_list.extend(img_list) images_crop_list.extend(crop_list) images_spatial_crop.extend(spatial_crop) tokenized_str.extend(tok_img) images_seq_mask.extend([True] * len(tok_img)) image_idx = image_idx + 1 if image_idx != len(images): raise ValueError( f"Data mismatch: Found {len(images)} images but only {image_idx} '' tokens were used." ) if not assistant_started: print( "Warning: No assistant message found in sample. Masking all tokens." ) prompt_token_count = len(tokenized_str) # Index to start training images_ori = torch.stack(images_list, dim=0) images_spatial_crop_tensor = torch.tensor( images_spatial_crop, dtype=torch.long ) if images_crop_list: images_crop = torch.stack(images_crop_list, dim=0) else: images_crop = torch.zeros( (1, 3, self.base_size, self.base_size), dtype=self.dtype ) return { "input_ids": torch.tensor(tokenized_str, dtype=torch.long), "images_seq_mask": torch.tensor(images_seq_mask, dtype=torch.bool), "images_ori": images_ori, "images_crop": images_crop, "images_spatial_crop": images_spatial_crop_tensor, "prompt_token_count": prompt_token_count, } def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]: """Collate batch of samples""" # --- 1. Setup --- batch_data = [] for feature in features: try: processed = self.process_single_sample(feature["messages"]) batch_data.append(processed) except Exception as e: print(f"Error processing sample: {e}") continue if not batch_data: raise ValueError("No valid samples in batch") input_ids_list = [item["input_ids"] for item in batch_data] images_seq_mask_list = [item["images_seq_mask"] for item in batch_data] prompt_token_counts = [item["prompt_token_count"] for item in batch_data] input_ids = pad_sequence( input_ids_list, batch_first=True, padding_value=self.tokenizer.pad_token_id, ) images_seq_mask = pad_sequence( images_seq_mask_list, batch_first=True, padding_value=False ) labels = input_ids.clone() # Index to start training labels[labels == self.tokenizer.pad_token_id] = -100 labels[images_seq_mask] = -100 if self.train_on_responses_only: for idx, prompt_count in enumerate( prompt_token_counts ): # Add BOS token at the very beginning if prompt_count > 0: labels[idx, :prompt_count] = -100 attention_mask = (input_ids != self.tokenizer.pad_token_id).long() images_batch = [] for item in batch_data: images_batch.append((item["images_crop"], item["images_ori"])) images_spatial_crop = torch.cat( [item["images_spatial_crop"] for item in batch_data], dim=0 ) return { "input_ids": input_ids, "attention_mask": attention_mask, "labels": labels, "images": images_batch, "images_seq_mask": images_seq_mask, "images_spatial_crop": images_spatial_crop, } # Check if this is the assistant's turn # This is the split point. All tokens added *so far* # are part of the prompt. # Append the EOS token string to the *end* of assistant content # Split this message's content by the image token # Tokenize the text part # If this text is followed by an tag # Process the image # Add image placeholder tokens # Move to the next image # --- 3. Validation and Final Prep --- # If we never found an assistant message, we're in a weird state # (e.g., user-only prompt). We mask everything. # Prepare image tensors # This is now accurate # Process each sample # Extract lists # Pad sequences # Create labels # Mask padding tokens # Mask image tokens (model shouldn't predict these) return (DeepSeekOCRDataCollator,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Train the model Now let's train our model. We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support `DPOTrainer` and `GRPOTrainer` for reinforcement learning!! We use our new `DeepSeekOCRDataCollator` which will help in our vision finetuning setup. """) return @app.cell def _( DeepSeekOCRDataCollator, FastVisionModel, converted_dataset, model_1, tokenizer, ): from transformers import Trainer, TrainingArguments from unsloth import is_bf16_supported FastVisionModel.for_training(model_1) # Enable for training! data_collator = DeepSeekOCRDataCollator( # Must use! tokenizer=tokenizer, model=model_1, image_size=640, base_size=1024, crop_mode=True, train_on_responses_only=True, ) trainer = Trainer( model=model_1, tokenizer=tokenizer, data_collator=data_collator, # Must use! train_dataset=converted_dataset, args=TrainingArguments( per_device_train_batch_size=2, gradient_accumulation_steps=4, warmup_steps=5, max_steps=60, learning_rate=0.0002, logging_steps=1, optim="adamw_8bit", weight_decay=0.001, lr_scheduler_type="linear", seed=3407, fp16=not is_bf16_supported(), # Use fp16 if bf16 is not supported bf16=is_bf16_supported(), # Use bf16 if supported output_dir="outputs", report_to="none", # For Weights and Biases dataloader_num_workers=2, remove_unused_columns=False, ), ) return (trainer,) @app.cell def _(torch): # @title Show current memory stats gpu_stats = torch.cuda.get_device_properties(0) start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") print(f"{start_gpu_memory} GB of memory reserved.") return max_memory, start_gpu_memory @app.cell def _(trainer): trainer_stats = trainer.train() return (trainer_stats,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Now after fine-tuning lets evaluate the model! """) return @app.cell def _( FastVisionModel, eval_dataset, evaluate_model, model_1, print_evaluation_summary, save_evaluation_results, tokenizer, ): FastVisionModel.for_inference(model_1) # Enable for inference! finetuned_results = evaluate_model( model=model_1, tokenizer=tokenizer, dataset=eval_dataset, num_samples=200, base_size=1024, image_size=640, crop_mode=True, verbose=True, ) print_evaluation_summary(finetuned_results, "Fine-tuned Model Performance") save_evaluation_results(finetuned_results, "finetuned_evaluation.json") return (finetuned_results,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Now lets compare both. """) return @app.cell def _(baseline_results, finetuned_results): print("\n" + "=" * 60) print("📈 PERFORMANCE COMPARISON") print("=" * 60) print(f"Baseline Mean CER: {baseline_results['mean_cer']:.2f}%") print(f"Fine-tuned Mean CER: {finetuned_results['mean_cer']:.2f}%") improvement = baseline_results["mean_cer"] - finetuned_results["mean_cer"] relative_improvement = (improvement / baseline_results["mean_cer"]) * 100 print(f"\n✨ Absolute Improvement: {improvement:.2f}%") print(f"✨ Relative Improvement: {relative_improvement:.2f}%") print("=" * 60) return @app.cell(hide_code=True) def _(mo): mo.md(r""" With only 60 steps, we reduced the Character Error Rate (CER) from 149.07% to 60.43%, representing a 88.6% absolute improvement in character errors! """) return @app.cell def _(max_memory, start_gpu_memory, torch, trainer_stats): # @title Show final memory and time stats used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) used_memory_for_lora = round(used_memory - start_gpu_memory, 3) used_percentage = round(used_memory / max_memory * 100, 3) lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") print( f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training." ) print(f"Peak reserved memory = {used_memory} GB.") print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") print(f"Peak reserved memory % of max memory = {used_percentage} %.") print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inference Let's run the model! """) return @app.cell def _(eval_dataset, model_1, tokenizer): eval_dataset[0]["image_path"].save("your_image.jpg") prompt = "\nFree OCR. " image_file = "your_image.jpg" output_path = "your/output/dir" # Tiny: base_size = 512, image_size = 512, crop_mode = False # Small: base_size = 640, image_size = 640, crop_mode = False # Base: base_size = 1024, image_size = 1024, crop_mode = False # Large: base_size = 1280, image_size = 1280, crop_mode = False # Gundam: base_size = 1024, image_size = 640, crop_mode = True res = model_1.infer( tokenizer, prompt=prompt, image_file=image_file, output_path=output_path, image_size=640, base_size=1024, crop_mode=True, save_results=True, test_compress=False, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving, loading finetuned models To save the final model as LoRA adapters, either use Hugging Face's `push_to_hub` for an online save or `save_pretrained` for a local save. **[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down! """) return @app.cell def _(model_1, tokenizer): model_1.save_pretrained("deepseek_ocr_lora") tokenizer.save_pretrained("deepseek_ocr_lora") return @app.cell(hide_code=True) def _(mo): mo.md(r""" Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`: """) return @app.cell def _(AutoModel, model_1, tokenizer): if False: from unsloth import FastVisionModel as _FastVisionModel _model, _tokenizer = _FastVisionModel.from_pretrained( model_name="deepseek_ocr_lora", # YOUR MODEL YOU USED FOR TRAINING load_in_4bit=False, # Use 4bit to reduce memory use. False for 16bit LoRA. auto_model=AutoModel, trust_remote_code=True, unsloth_force_compile=True, use_gradient_checkpointing="unsloth", # True or "unsloth" for long context ) _FastVisionModel.for_inference(_model) prompt_1 = "\nFree OCR. " image_file_1 = "your_image.jpg" output_path_1 = "your/output/dir" # Tiny: base_size = 512, image_size = 512, crop_mode = False # Small: base_size = 640, image_size = 640, crop_mode = False # Base: base_size = 1024, image_size = 1024, crop_mode = False # Large: base_size = 1280, image_size = 1280, crop_mode = False # Gundam: base_size = 1024, image_size = 640, crop_mode = True res_1 = model_1.infer( tokenizer, prompt=prompt_1, image_file=image_file_1, output_path=output_path_1, image_size=640, base_size=1024, crop_mode=True, save_results=True, test_compress=False, ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Saving to float16 for VLLM We also support saving to `float16` directly. Select `merged_16bit` for float16. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens. See [our docs](https://unsloth.ai/docs/basics/inference-and-deployment) for more deployment options. """) return @app.cell def _(model_1, tokenizer): # Select ONLY 1 to save! (Both not needed!) if False: # Save locally to 16bit model_1.save_pretrained_merged("unsloth_finetune", tokenizer) if False: # To export and save to your Hugging Face account model_1.push_to_hub_merged( "YOUR_USERNAME/unsloth_finetune", tokenizer, token="YOUR_HF_TOKEN" ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord! Some other resources: 1. Train your own reasoning model - Llama GRPO notebook [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb) 2. Saving finetunes to Ollama. [Free notebook](https://github.com/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb) 3. Llama 3.2 Vision finetuning - Radiography use case. [Open in molab](https://github.com/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb) 4. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://unsloth.ai/docs/get-started/unsloth-notebooks)!

Join Discord if you need help + ⭐️ Star us on Github ⭐️ This notebook and all Unsloth notebooks are licensed [LGPL-3.0](https://github.com/unslothai/notebooks?tab=LGPL-3.0-1-ov-file#readme)

""") return if __name__ == "__main__": app.run()