.env

# compose sources this file
# docs: https://docs.docker.com/compose/environment-variables/set-environment-variables/
LLAMA_CPP_SERVER_PORT=8000
LLAMA_CPP_HOST=llama-cpp-server
LOGDETECTIVE_SERVER_PORT=8080
# for some reason, fastapi cripples sys.path and some deps cannot be found
PYTHONPATH=/src:/usr/local/lib64/python3.12/site-packages:/usr/lib64/python312.zip:/usr/lib64/python3.12/:/usr/lib64/python3.12/lib-dynload:/usr/local/lib/python3.12/site-packages:/usr/lib64/python3.12/site-packages:/usr/lib/python3.12/site-packages
LLAMA_ARG_MODEL="/models/mistral-7b-instruct-v0.2.Q4_K_S.gguf"
LLAMA_ARG_ALIAS="default-model"
# -1 is syntax of python-llama
LLAMA_ARG_N_GPU_LAYERS=64
LLAMA_ARG_THREADS=12
LLAMA_ARG_BATCH=512
# Modify following var when switching model
LLAMA_ARG_CHAT_TEMPLATE="mistral-v3"
LLAMA_ARG_CTX_SIZE=32768
LLAMA_ARG_N_PARALLEL=4