NVIDIA NeMo Inference Microservice デプロイジョブ

net: host
gpus: all # 特定の GPU セット、またはすべてを使用する場合は `all` を指定できます
runtime: nvidia # nvidia container runtime も必要です
volume:
  - model-store:/model-store/

wandb job create -n "deploy-to-nvidia-nemo-inference-microservice" \
   -e $ENTITY \
   -p $PROJECT \
   -E jobs/deploy_to_nvidia_nemo_inference_microservice/job.py \
   -g andrew/nim-updates \
   git https://github.com/wandb/launch-jobs

wandb launch-agent -e $ENTITY -p $PROJECT -q $QUEUE

 #!/bin/bash
 curl -X POST "http://0.0.0.0:9999/v1/completions" \
     -H "accept: application/json" \
     -H "Content-Type: application/json" \
     -d '{
         "model": "ensemble",
         "prompt": "Tell me a joke",
         "max_tokens": 256,
         "temperature": 0.5,
         "n": 1,
         "stream": false,
         "stop": "string",
         "frequency_penalty": 0.0
         }'