henryx/haisgl repository overview

amd/Meta-Llama-3.1-405B-Instruct-FP8-KV
amd/Meta-Llama-3.1-70B-Instruct-FP8-KV
amd/Meta-Llama-3.1-8B-Instruct-FP8-KV
amd/dbrx-instruct-FP8-KV
amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV
amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV
neuralmagic/Llama-2-7b-chat-hf-FP8
neuralmagic/Mistral-7B-Instruct-v0.3-FP8
neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8
neuralmagic/Qwen2-0.5B-Instruct-FP8
neuralmagic/Qwen2-1.5B-Instruct-FP8
neuralmagic/Qwen2-7B-Instruct-FP8
neuralmagic/Meta-Llama-3-8B-Instruct-FP8
neuralmagic/Meta-Llama-3-70B-Instruct-FP8

alias drun='sudo docker run -it --rm --network=host --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v $HOME/dockerx:/dockerx -v /data:/data -v /data2:/data2'

root@x:/dockerx# ln -s /sglang ./sglang
root@x:/dockerx# cd sglang
root@x:/dockerx/sglang#
root@x:/dockerx/sglang# MOE_PADDING=1 python -m sglang.bench_latency --batch-size 32 --input 1024 --output 8 --model dummy_grok1/ --tokenizer-path Xenova/grok-1-tokenizer --load-format dummy --tp 8 --quantization fp8

root@x:/dockerx/sglang# cat dummy_grok1/config.json 
{
  "architectures": [
    "Grok1ModelForCausalLM"
  ],
  "embedding_multiplier_scale": 78.38367176906169,
  "output_multiplier_scale": 0.5773502691896257,
  "vocab_size": 131072,
  "hidden_size": 6144,
  "intermediate_size": 32768,
  "max_position_embeddings": 8192,
  "num_experts_per_tok": 2,
  "num_local_experts": 8,
  "num_attention_heads": 48,
  "num_hidden_layers": 64,
  "num_key_value_heads": 8,
  "head_dim": 128,
  "rms_norm_eps": 1e-05,
  "rope_theta": 10000.0,
  "model_type": "mixtral",
  "torch_dtype": "bfloat16"
}