Hire Expert Hugging Face
Developers from India

from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline from peft import LoraConfig, get_peft_model import torch # Load base model with 4-bit quantization model = AutoModelForCausalLM.from_pretrained( "mistralai/Mistral-7B-Instruct-v0.3", load_in_4bit=True, device_map="auto", torch_dtype=torch.float16, ) # Add LoRA adapter for fine-tuning lora_config = LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], lora_dropout=0.05) model = get_peft_model(model, lora_config) model.print_trainable_parameters() # ~0.1% of total params # Inference pipeline tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3") pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=256)