Files

Here is a listing of the various configuration files available out-of-the-box with FMBench. Click on any link to view a file. You can use these files as-is or use them as templates to create a custom configuration file for your use-case of interest.

bedrock
├── bedrock/config-bedrock-all-anthropic-models-longbench-data.yml
├── bedrock/config-bedrock-anthropic-models-OpenOrca.yml
├── bedrock/config-bedrock-claude.yml
├── bedrock/config-bedrock-evals-only-conc-1.yml
├── bedrock/config-bedrock-haiku-sonnet-majority-voting.yml
├── bedrock/config-bedrock-llama3-1-70b-streaming.yml
├── bedrock/config-bedrock-llama3-1-8b-streaming.yml
├── bedrock/config-bedrock-llama3-1-no-streaming.yml
├── bedrock/config-bedrock-llama3-streaming.yml
├── bedrock/config-bedrock-models-OpenOrca.yml
├── bedrock/config-bedrock-titan-text-express.yml
├── bedrock/config-bedrock.yml
├── bedrock/config-llama-3-2-all-models.yml
└── bedrock/config-llama-3-2-vision-instruct.yml
bert
└── bert/config-distilbert-base-uncased.yml
byoe
└── byoe/config-model-byo-sagemaker-endpoint.yml
eks_manifests
├── eks_manifests/llama3-ray-service.yaml
└── eks_manifests/mistral-ray-service.yaml
embeddings
├── embeddings/bge-base-en-v1-5-c5-embeddings.yml
├── embeddings/bge-base-en-v1-5-g5-embeddings.yml
└── embeddings/bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml
gemma
└── gemma/config-gemma-2b-g5.yml
llama2
├── llama2/13b
│   ├── llama2/13b/config-bedrock-sagemaker-llama2.yml
│   ├── llama2/13b/config-byo-rest-ep-llama2-13b.yml
│   ├── llama2/13b/config-llama2-13b-inf2-g5-p4d.yml
│   └── llama2/13b/config-llama2-13b-inf2-g5.yml
├── llama2/70b
│   ├── llama2/70b/config-ec2-llama2-70b.yml
│   ├── llama2/70b/config-llama2-70b-g5-p4d-tgi.yml
│   ├── llama2/70b/config-llama2-70b-g5-p4d-trt.yml
│   └── llama2/70b/config-llama2-70b-inf2-g5.yml
└── llama2/7b
├── llama2/7b/config-llama2-7b-byo-sagemaker-endpoint.yml
├── llama2/7b/config-llama2-7b-g4dn-g5-trt.yml
├── llama2/7b/config-llama2-7b-g5-no-s3-quick.yml
├── llama2/7b/config-llama2-7b-g5-quick.yml
└── llama2/7b/config-llama2-7b-inf2-g5.yml
llama3
├── llama3/70b
│   ├── llama3/70b/config-bedrock.yml -> ../../bedrock/config-bedrock.yml
│   ├── llama3/70b/config-ec2-llama3-70b-instruct.yml
│   ├── llama3/70b/config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml
│   ├── llama3/70b/config-llama3-70b-instruct-g5-48xl.yml
│   ├── llama3/70b/config-llama3-70b-instruct-g5-p4d.yml
│   └── llama3/70b/config-llama3-70b-instruct-p4d.yml
└── llama3/8b
├── llama3/8b/config-bedrock.yml
├── llama3/8b/config-ec2-llama3-8b-c5-18xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-g6e-2xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-inf2-48xl.yml
├── llama3/8b/config-ec2-llama3-8b-m5-16xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7a-16xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7a-24xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7i-12xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml
├── llama3/8b/config-ec2-llama3-8b.yml
├── llama3/8b/config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml
├── llama3/8b/config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml
├── llama3/8b/config-llama3-8b-eks-inf2.yml
├── llama3/8b/config-llama3-8b-g5-streaming.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml
├── llama3/8b/config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml
├── llama3/8b/config-llama3-8b-inf2-48xlarge-triton-djl.yml
├── llama3/8b/config-llama3-8b-inf2-g5-byoe-w-openorca.yml
├── llama3/8b/config-llama3-8b-inf2-g5.yml
├── llama3/8b/config-llama3-8b-instruct-all.yml
├── llama3/8b/config-llama3-8b-instruct-g5-12xl-4-instances.yml
├── llama3/8b/config-llama3-8b-instruct-g5-12xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-24xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-2xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-48xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-p4d.yml
├── llama3/8b/config-llama3-8b-instruct-g6-12xl.yml
├── llama3/8b/config-llama3-8b-instruct-g6-24xl.yml
├── llama3/8b/config-llama3-8b-instruct-g6-48xl.yml
├── llama3/8b/config-llama3-8b-instruct-p4d-djl-lmi-dist.yml
├── llama3/8b/config-llama3-8b-instruct-p4d-djl-vllm.yml
├── llama3/8b/config-llama3-8b-instruct-p5-djl-lmi-dist.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml
├── llama3/8b/config-llama3-8b-trn1-32xlarge-triton-djl.yml
├── llama3/8b/config-llama3-8b-trn1-32xlarge-triton-vllm.yml
├── llama3/8b/config-llama3-8b-trn1.yml
├── llama3/8b/llama3-8b-inf2-24xl-byoe-g5-12xl.yml
├── llama3/8b/llama3-8b-inf2-48xl-byoe-g5-24xl.yml
└── llama3/8b/llama3-8b-trn1-32xl-byoe-g5-24xl.yml
llama3.1
├── llama3.1/70b
│   ├── llama3.1/70b/config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml
│   ├── llama3.1/70b/config-ec2-llama3-1-70b-inf2-deploy-sm.yml
│   ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml
│   ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml
│   ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml
│   ├── llama3.1/70b/config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml
│   ├── llama3.1/70b/config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml
│   ├── llama3.1/70b/config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml
│   ├── llama3.1/70b/config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml
│   ├── llama3.1/70b/config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml
│   ├── llama3.1/70b/config-llama3-1-70b-inf2.48xl-triton-tp24.yml
│   ├── llama3.1/70b/config-llama3-1-70b-p5-djl-lmi.yml
│   ├── llama3.1/70b/config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml
│   └── llama3.1/70b/config-llama3-1-7b-inf2.48xl-triton-ec2.yml
└── llama3.1/8b
├── llama3.1/8b/client-config-ec2-llama3-1-8b.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml
├── llama3.1/8b/config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml
├── llama3.1/8b/config-llama3.1-8b-g5-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-trn32xl-triton-vllm.yml
└── llama3.1/8b/server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
llama3.2
├── llama3.2/config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml
├── llama3.2/config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml
└── llama3.2/config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml
mistral
├── mistral/config-mistral-7b-eks-inf2.yml
├── mistral/config-mistral-7b-tgi-g5.yml
├── mistral/config-mistral-7b-trn1-32xl-triton.yml
├── mistral/config-mistral-instruct-AWQ-p4d.yml
├── mistral/config-mistral-instruct-AWQ-p5-byo-ep.yml
├── mistral/config-mistral-instruct-AWQ-p5.yml
├── mistral/config-mistral-instruct-p4d.yml
├── mistral/config-mistral-instruct-v1-p5-trtllm.yml
├── mistral/config-mistral-instruct-v2-p4d-lmi-dist.yml
├── mistral/config-mistral-instruct-v2-p4d-trtllm.yml
├── mistral/config-mistral-instruct-v2-p5-lmi-dist.yml
├── mistral/config-mistral-instruct-v2-p5-trtllm.yml
├── mistral/config-mistral-trn1-32xl-deploy-ec2-tp32.yml
└── mistral/config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml
mixtral
└── mixtral/config-mixtral-8x7b-g6e.48xl-ec2.yml
model_eval_all_info.yml
phi
└── phi/config-phi-3-g5.yml
pricing.yml