Files
Here is a listing of the various configuration files available out-of-the-box with FMBench. Click on any link to view a file. You can use these files as-is or use them as templates to create a custom configuration file for your use-case of interest.
NousResearchHermes70B
├── NousResearchHermes70B/config-Nous-Hermes3-1-70b-g5.48xl-tp-8-mc-max-djl.yml
├── NousResearchHermes70B/config-Nous-Hermes3-1-70b-inf2.48xl-triton-tp24.yml
└── NousResearchHermes70B/config-Nous-Hermes3-1-70b-trn1.32xl-deploy-ec2-triton.yml
bedrock
├── bedrock/config-bedrock-all-anthropic-models-longbench-data.yml
├── bedrock/config-bedrock-anthropic-models-OpenOrca.yml
├── bedrock/config-bedrock-claude.yml
├── bedrock/config-bedrock-evals-only-conc-1.yml
├── bedrock/config-bedrock-haiku-sonnet-majority-voting.yml
├── bedrock/config-bedrock-llama3-1-70b-streaming.yml
├── bedrock/config-bedrock-llama3-1-8b-streaming.yml
├── bedrock/config-bedrock-llama3-1-no-streaming.yml
├── bedrock/config-bedrock-llama3-1.yml
├── bedrock/config-bedrock-llama3-streaming.yml
├── bedrock/config-bedrock-models-OpenOrca.yml
├── bedrock/config-bedrock-titan-text-express.yml
├── bedrock/config-bedrock.yml
├── bedrock/config-claude-3-5-sonnet-v2.yml
├── bedrock/config-claude-dolly-dataset.yml
├── bedrock/config-llama-3-2-11b-databricks-dolly-15k.yml
├── bedrock/config-llama-3-2-1b-3b-no-evals.yml
├── bedrock/config-llama-3-2-1b-3b.yml
├── bedrock/config-llama-3-2-all-models-longbench-hf-version.yml
├── bedrock/config-llama-3-2-all-models.yml
└── bedrock/config-nova-all-models.yml
bert
└── bert/config-distilbert-base-uncased.yml
byoe
└── byoe/config-model-byo-sagemaker-endpoint.yml
eks_manifests
├── eks_manifests/llama3-ray-service.yaml
└── eks_manifests/mistral-ray-service.yaml
embeddings
├── embeddings/bge-base-en-v1-5-c5-embeddings.yml
├── embeddings/bge-base-en-v1-5-g5-embeddings.yml
└── embeddings/bge-base-en-v1-5-g5-g4dn-c7-embeddings.yml
gemma
└── gemma/config-gemma-2b-g5.yml
llama2
├── llama2/13b
│ ├── llama2/13b/config-bedrock-sagemaker-llama2.yml
│ ├── llama2/13b/config-byo-rest-ep-llama2-13b.yml
│ ├── llama2/13b/config-llama2-13b-inf2-g5-p4d.yml
│ └── llama2/13b/config-llama2-13b-inf2-g5.yml
├── llama2/70b
│ ├── llama2/70b/config-ec2-llama2-70b.yml
│ ├── llama2/70b/config-llama2-70b-g5-p4d-tgi.yml
│ ├── llama2/70b/config-llama2-70b-g5-p4d-trt.yml
│ └── llama2/70b/config-llama2-70b-inf2-g5.yml
└── llama2/7b
├── llama2/7b/config-llama2-7b-byo-sagemaker-endpoint.yml
├── llama2/7b/config-llama2-7b-g4dn-g5-trt.yml
├── llama2/7b/config-llama2-7b-g5-no-s3-quick.yml
├── llama2/7b/config-llama2-7b-g5-quick.yml
└── llama2/7b/config-llama2-7b-inf2-g5.yml
llama3
├── llama3/70b
│ ├── llama3/70b/config-bedrock.yml -> ../../bedrock/config-bedrock.yml
│ ├── llama3/70b/config-ec2-llama3-70b-instruct.yml
│ ├── llama3/70b/config-ec2-neuron-llama3-70b-inf2-48xl-deploy-sm.yml
│ ├── llama3/70b/config-llama3-70b-instruct-g5-48xl.yml
│ ├── llama3/70b/config-llama3-70b-instruct-g5-p4d.yml
│ └── llama3/70b/config-llama3-70b-instruct-p4d.yml
└── llama3/8b
├── llama3/8b/config-bedrock.yml
├── llama3/8b/config-ec2-llama3-8b-c5-18xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-g6e-2xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-inf2-48xl.yml
├── llama3/8b/config-ec2-llama3-8b-m5-16xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7a-16xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7a-24xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7i-12xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7i-16xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-m7i-24xlarge.yml
├── llama3/8b/config-ec2-llama3-8b-neuron-trn1-32xl-tp16-sm.yml
├── llama3/8b/config-ec2-llama3-8b-p4d-tp-2-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p4d-tp-4-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p4d-tp-8-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p4de-tp-2-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p4de-tp-4-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p4de-tp-8-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p5-tp-2-mc-max.yml
├── llama3/8b/config-ec2-llama3-8b-p5-tp-8-mc-auto.yml
├── llama3/8b/config-ec2-llama3-8b.yml
├── llama3/8b/config-ec2-neuron-llama3-8b-inf2-24xl-deploy-sm.yml
├── llama3/8b/config-ec2-neuron-llama3-8b-inf2-48xl-deploy-sm.yml
├── llama3/8b/config-llama3-8b-eks-inf2.yml
├── llama3/8b/config-llama3-8b-g5-streaming.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g5.12xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g5.48xl-tp-8-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g5.4xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.12xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.24xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.2xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-2-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-4-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.48xl-tp-8-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-djl-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.4xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-g6e.xl-tp-1-mc-max-triton-ec2.yml
├── llama3/8b/config-llama3-8b-inf2-24xl-tp=8-bs=4-byoe.yml
├── llama3/8b/config-llama3-8b-inf2-48xl-tp=8-bs=4-byoe.yml
├── llama3/8b/config-llama3-8b-inf2-48xlarge-triton-djl.yml
├── llama3/8b/config-llama3-8b-inf2-g5-byoe-w-openorca.yml
├── llama3/8b/config-llama3-8b-inf2-g5.yml
├── llama3/8b/config-llama3-8b-instruct-all.yml
├── llama3/8b/config-llama3-8b-instruct-g5-12xl-4-instances.yml
├── llama3/8b/config-llama3-8b-instruct-g5-12xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-24xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-2xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-48xl.yml
├── llama3/8b/config-llama3-8b-instruct-g5-p4d.yml
├── llama3/8b/config-llama3-8b-instruct-g6-12xl.yml
├── llama3/8b/config-llama3-8b-instruct-g6-24xl.yml
├── llama3/8b/config-llama3-8b-instruct-g6-48xl.yml
├── llama3/8b/config-llama3-8b-instruct-p4d-djl-lmi-dist.yml
├── llama3/8b/config-llama3-8b-instruct-p4d-djl-vllm.yml
├── llama3/8b/config-llama3-8b-instruct-p5-djl-lmi-dist.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp-16-bs-4-byoe.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp-8-bs-4-byoe.yml
├── llama3/8b/config-llama3-8b-trn1-32xl-tp16-bs-4-ec2.yml
├── llama3/8b/config-llama3-8b-trn1-32xlarge-triton-djl.yml
├── llama3/8b/config-llama3-8b-trn1-32xlarge-triton-vllm.yml
├── llama3/8b/config-llama3-8b-trn1.yml
├── llama3/8b/llama3-8b-inf2-24xl-byoe-g5-12xl.yml
├── llama3/8b/llama3-8b-inf2-48xl-byoe-g5-24xl.yml
└── llama3/8b/llama3-8b-trn1-32xl-byoe-g5-24xl.yml
llama3.1
├── llama3.1/70b
│ ├── llama3.1/70b/config-ec2-llama3-1-70b-inf2-48xl-deploy-ec2-djl.yml
│ ├── llama3.1/70b/config-ec2-llama3-1-70b-inf2-deploy-sm.yml
│ ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-large-prompts.yml
│ ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-longbench.yml
│ ├── llama3.1/70b/config-ec2-llama3-1-70b-p4de.24xl-deploy-ec2-summarization.yml
│ ├── llama3.1/70b/config-llama3-1-70b-g5.48xl-tp-8-mc-max-djl.yml
│ ├── llama3.1/70b/config-llama3-1-70b-g6.48xl-tp-8-mc-max-djl.yml
│ ├── llama3.1/70b/config-llama3-1-70b-g6e.24xl-tp-4-mc-max-djl.yml
│ ├── llama3.1/70b/config-llama3-1-70b-g6e.48xl-tp-8-mc-max-djl.yml
│ ├── llama3.1/70b/config-llama3-1-70b-inf2.48xl-deploy-ec2-triton.yml
│ ├── llama3.1/70b/config-llama3-1-70b-inf2.48xl-triton-tp24.yml
│ ├── llama3.1/70b/config-llama3-1-70b-p5-djl-lmi.yml
│ ├── llama3.1/70b/config-llama3-1-70b-trn1.32xl-deploy-ec2-triton.yml
│ └── llama3.1/70b/config-llama3-1-7b-inf2.48xl-triton-ec2.yml
└── llama3.1/8b
├── llama3.1/8b/client-config-ec2-llama3-1-8b.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-g6e-2xlarge-byoe-ollama.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2-tp24-bs12.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-inf2.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-2-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-4-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p4-tp-8-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-p5-tp-2-mc-max.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-tp-8-mc-auto-p5.yml
├── llama3.1/8b/config-ec2-llama3-1-8b-trn1-32xl-deploy-ec2-tp32-bs8.yml
├── llama3.1/8b/config-llama3-1-8b-g5.12xlarge-djl-lmi-sm.yml
├── llama3.1/8b/config-llama3.1-8b-g5-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.12xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.24xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml
├── llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2-conc-1-2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.2xl-tp-1-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-auto-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-2-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-4-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.48xl-tp-8-mc-max-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.xl-tp-1-mc-max-ec2-conc-1-2.yml
├── llama3.1/8b/config-llama3.1-8b-g5.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.12xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.24xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.2xl-tp-1-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-2-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-4-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.48xl-tp-8-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-g6e.4xl-tp-1-mc-max-djl.yml
├── llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-24-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-inf2-48xl-deploy-tp-8-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-trn1-32xl-deploy-tp-8-ec2.yml
├── llama3.1/8b/config-llama3.1-8b-trn32xl-triton-vllm.yml
└── llama3.1/8b/server-config-ec2-llama3-1-8b-inf2-48xl-deploy-ec2.yml
llama3.2
├── llama3.2/11b
│ └── llama3.2/11b/config-llama3.2-11b-g6e-2xl-tp-1-mc-max-djl-vllm-ec2.yml
├── llama3.2/1b
│ ├── llama3.2/1b/config-llama3.2-1b-g5.2xl-summarization-500-50.yml
│ ├── llama3.2/1b/config-llama3.2-1b-g5.2xl-tp-1-mc-max-djl-ec2.yml
│ ├── llama3.2/1b/config-llama3.2-1b-g5.4xl-tp-1-mc-max-djl-ec2.yml
│ ├── llama3.2/1b/config-llama3.2-1b-g6e.2xl-tp-1-mc-max-djl.yml
│ ├── llama3.2/1b/config-llama3.2-1b-m5-16xlarge-ec2.yml
│ ├── llama3.2/1b/config-llama3.2-1b-m7a-16xlarge-ec2.yml
│ ├── llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2-summarization.yml
│ ├── llama3.2/1b/config-llama3.2-1b-m7a-24xlarge-ec2.yml
│ └── llama3.2/1b/config-llama3.2-1b-m7i-12xlarge-ec2.yml
└── llama3.2/3b
└── llama3.2/3b/config-llama3.2-3b-g5.4xl-tp-1-mc-max-djl-ec2.yml
mistral
├── mistral/config-mistral-7b-eks-inf2.yml
├── mistral/config-mistral-7b-tgi-g5.yml
├── mistral/config-mistral-7b-trn1-32xl-triton.yml
├── mistral/config-mistral-instruct-AWQ-p4d.yml
├── mistral/config-mistral-instruct-AWQ-p5-byo-ep.yml
├── mistral/config-mistral-instruct-AWQ-p5.yml
├── mistral/config-mistral-instruct-p4d.yml
├── mistral/config-mistral-instruct-v1-p5-trtllm.yml
├── mistral/config-mistral-instruct-v2-p4d-lmi-dist.yml
├── mistral/config-mistral-instruct-v2-p4d-trtllm.yml
├── mistral/config-mistral-instruct-v2-p5-lmi-dist.yml
├── mistral/config-mistral-instruct-v2-p5-trtllm.yml
├── mistral/config-mistral-trn1-32xl-deploy-ec2-tp32.yml
└── mistral/config-mistral-v3-inf2-48xl-deploy-ec2-tp24.yml
mixtral
└── mixtral/config-mixtral-8x7b-g6e.48xl-ec2.yml
model_eval_all_info.yml
multimodal
└── multimodal/bedrock
├── multimodal/bedrock/config-claude-scienceqa.yml
├── multimodal/bedrock/config-llama-3-2-11b-vision-instruct-marqo-GS-10M.yml
├── multimodal/bedrock/config-llama-3-2-11b-vision-instruct-scienceqa.yml
└── multimodal/bedrock/config-llama-3-2-claude-models-scienceqa.yml
phi
└── phi/config-phi-3-g5.yml
pricing.yml
pricing_fallback.yml