[](https://hamel.dev/notes/llm/inference/inference.html#cb16-1)from vllm import SamplingParams, LLM
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-2)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-3)#from https://modal.com/docs/guide/ex/vllm_inference
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-4)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-5)questions = [
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-6)    # Coding questions
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-7)    "Implement a Python function to compute the Fibonacci numbers.",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-8)    "Write a Rust function that performs binary exponentiation.",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-9)    "What are the differences between Javascript and Python?",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-10)    # Literature
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-11)    "Write a story in the style of James Joyce about a trip to the Australian outback in 2083, to see robots in the beautiful desert.",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-12)    "Who does Harry turn into a balloon?",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-13)    "Write a tale about a time-traveling historian who's determined to witness the most significant events in human history.",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-14)    # Math
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-15)    "What is the product of 9 and 8?",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-16)    "If a train travels 120 kilometers in 2 hours, what is its average speed?",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-17)    "Think through this step by step. If the sequence a_n is defined by a_1 = 3, a_2 = 5, and a_n = a_(n-1) + a_(n-2) for n > 2, find a_6.",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-18)]
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-19)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-20)MODEL_DIR = "/home/ubuntu/hamel-drive/vllm-models"
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-21)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-22)def download_model_to_folder():
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-23)    from huggingface_hub import snapshot_download
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-24)    import os
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-25)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-26)    snapshot_download(
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-27)        "meta-llama/Llama-2-7b-hf",
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-28)        local_dir=MODEL_DIR,
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-29)        token=os.environ["HUGGING_FACE_HUB_TOKEN"],
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-30)    )
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-31)    return LLM(MODEL_DIR)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-32)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-33)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-34)def generate(question, llm, note=None):
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-35)    response = {'question': question, 'note': note}
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-36)    sampling_params = SamplingParams(
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-37)        temperature=1.0,
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-38)        top_p=1,
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-39)        max_tokens=200,
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-40)    )
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-41)    
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-42)    start = time.perf_counter()
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-43)    result = llm.generate(question, sampling_params)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-44)    request_time = time.perf_counter() - start
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-45)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-46)    for output in result:
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-47)        response['tok_count'] = len(output.outputs[0].token_ids)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-48)        response['time'] = request_time
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-49)        response['answer'] = output.outputs[0].text
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-50)    
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-51)    return response
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-52)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-53)if __name__ == '__main__':
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-54)    llm = download_model_to_folder()
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-55)    counter = 1
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-56)    responses = []
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-57)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-58)    for q in questions:
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-59)        response = generate(question=q, llm=llm, note='vLLM')
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-60)        if counter >= 2:
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-61)            responses.append(response)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-62)        counter += 1
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-63)    
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-64)    df = pd.DataFrame(responses)
[](https://hamel.dev/notes/llm/inference/inference.html#cb16-65)    df.to_csv('bench-vllm.csv', index=False)