import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
# path to gptq weights
model_id = "quantized_llama"
 
q_tokenizer = AutoTokenizer.from_pretrained(model_id)
q_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.float16)
 
qtq_pipe = pipeline("text-generation", model=q_model, tokenizer=q_tokenizer)