from vllm import LLM, SamplingParams

llm = LLM(
    model="Qwen/Qwen2.5-72B-Instruct",
    speculative_model="Qwen/Qwen2.5-0.5B-Instruct",
    num_speculative_tokens=5,
    tensor_parallel_size=4,
)

outputs = llm.generate(["Explain speculative decoding."],
                       SamplingParams(temperature=0, max_tokens=256))