[](https://hamel.dev/notes/llm/inference/inference.html#cb7-1)#!/bin/bash
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-2)
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-3)if [ -z "$HUGGING_FACE_HUB_TOKEN" ]
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-4)then
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-5)  echo "HUGGING_FACE_HUB_TOKEN is not set. Please set it before running this script."
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-6)  exit 1
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-7)fi
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-8)
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-9)model="TheBloke/Llama-2-7B-GPTQ"
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-10)volume=$PWD/data
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-11)
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-12)docker run --gpus all \
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-13) -e HUGGING_FACE_HUB_TOKEN=$HUGGING_FACE_HUB_TOKEN \
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-14) -e GPTQ_BITS=4 -e GPTQ_GROUPSIZE=128 \
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-15) --shm-size 5g -p 8081:80 \
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-16) -v $volume:/data ghcr.io/huggingface/text-generation-inference \
[](https://hamel.dev/notes/llm/inference/inference.html#cb7-17) --max-best-of 1 "$@"