TensorRT

TensorRT

High performance inference engine.
pertama-tama download tar.gz-nya di https://developer.nvidia.com/tensorrt-rtx

tar -xf TensorRT-RTX-1.1.1.26.Linux.x86_64-gnu.cuda-12.9.tar.gz
cd TensorRT-RTX-1.1.1.26
export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH # masukin .bashrc atau .zshrc
sudo ln -s $(realpath bin/tensorrt_rtx) /usr/local/bin/trtexec

Convert

# Konversi dasar ONNX ke TensorRT
trtexec --onnx=mobilenetv2-7.onnx --saveEngine=mobilenetv2.trt

# Konversi dengan optimasi tambahan (direkomendasikan)
trtexec --onnx=mobilenetv2-7.onnx \
        --saveEngine=mobilenetv2_optimized.trt \
        --minShapes=data:1x3x224x224 \
        --optShapes=data:1x3x224x224 \
        --maxShapes=data:1x3x224x224 \
        --builderOptimizationLevel=5 \
        --avgTiming=16 \
        --memPoolSize=workspace:2G \
        --verbose

# Konversi dengan dynamic shapes (jika perlu batch size fleksibel)
trtexec --onnx=mobilenetv2-7.onnx \
        --saveEngine=mobilenetv2_dynamic.trt \
        --minShapes=data:1x3x224x224 \
        --optShapes=data:4x3x224x224 \
        --maxShapes=data:16x3x224x224 \
        --builderOptimizationLevel=5 \
        --verbose

# Konversi dengan precision FP16 untuk performa lebih cepat (jika GPU mendukung)
trtexec --onnx=mobilenetv2-7.onnx \
        --saveEngine=mobilenetv2_fp16.trt \
        --minShapes=data:1x3x224x224 \
        --optShapes=data:1x3x224x224 \
        --maxShapes=data:1x3x224x224 \
        --fp16 \
        --builderOptimizationLevel=5 \
        --verbose

# Konversi dengan hardware compatibility (untuk portabilitas GPU)
trtexec --onnx=mobilenetv2-7.onnx \
        --saveEngine=mobilenetv2_compatible.trt \
        --minShapes=data:1x3x224x224 \
        --optShapes=data:1x3x224x224 \
        --maxShapes=data:1x3x224x224 \
        --hardwareCompatibilityLevel=ampere+ \
        --verbose

# Test inference setelah konversi
trtexec --loadEngine=mobilenet.trt \
        --shapes=data:1x3x224x224 \
        --iterations=100 \
        --avgRuns=10 \
        --exportOutput=test_result.json \
        --verbose

Triton Server

# docker-compose.yml
version: "3.9"

services:
  tritonserver:
    image: nvcr.io/nvidia/tritonserver:24.10-py3
    container_name: tritonserver
    restart: unless-stopped
    shm_size: 1g
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    command: >
      tritonserver
        --model-repository=/models
        --log-verbose=1
        --allow-grpc=true
        --grpc-port=50001
        --allow-http=true
        --http-port=50000
        --metrics-port=50002
    volumes:
      - ./model_repository:/models
    ports:
      - "0.0.0.0:50000:50000"   # HTTP/REST
      - "0.0.0.0:50001:50001"   # gRPC
      - "0.0.0.0:50002:50002"   # Metrics (Prometheus)

  prometheus:
    image: prom/prometheus
    container_name: prometheus
    restart: unless-stopped
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "0.0.0.0:50004:9090"   # Prometheus UI

  grafana:
    image: grafana/grafana
    container_name: grafana
    restart: unless-stopped
    ports:
      - "0.0.0.0:50005:3000"   # Grafana UI
    volumes:
      - grafana-data:/var/lib/grafana

volumes:
  grafana-data:

# prometheus.yml
global:
  scrape_interval: 5s

scrape_configs:
  - job_name: "triton"
    static_configs:
      - targets: ["tritonserver:50002"]

Struktur direktori model_repository

model_repository/
└── brand_equity
    ├── 1
    │   └── model.plan
    └── config.pbtxt

contoh config.pbtxt

name: "my_model"
platform: "tensorrt_plan"
max_batch_size: 10

input [
  {
    name: "input_ids"
    data_type: TYPE_INT64
    dims: [128]
  },
  {
    name: "attention_mask"
    data_type: TYPE_INT64
    dims: [128]
  }
]

output [
  {
    name: "dimensi_logits"
    data_type: TYPE_FP32
    dims: [6]
  },
  {
    name: "subdimensi_logits"
    data_type: TYPE_FP32
    dims: [26]
  }
]

instance_group [
  {
    kind: KIND_GPU
    count: 1
  }
]

dynamic_batching {
  preferred_batch_size: [4, 10]
  max_queue_delay_microseconds: 5000
}

Triton Server Data Type

Model Config	TensorRT	ONNX Runtime	PyTorch	API	NumPy
TYPE_BOOL	kBOOL	BOOL	kBool	BOOL	bool
TYPE_UINT8	kUINT8	UINT8	kByte	UINT8	uint8
TYPE_UINT16	-	UINT16	-	UINT16	uint16
TYPE_UINT64	-	UINT64	-	UINT64	uint16
TYPE_INT8	kINT8	INT8	kChar	INT8	int8
TYPE_INT16	-	INT16	kShort	INT16	int16
TYPE_INT32	kINT32	INT32	kInt	INT32	int32
TYPE_INT64	kINT64	kINT64	kLong	INT64	int64
TYPE_FP16	kHALF	FLOAT16	-	FP16	float16
TYPE_FP32	kFLOAT	FLOAT	kFloat	FP32	float32
TYPE_FP64	-	DOUBLE	kDouble	FP64	float64
TYPE_STRING	-	STRING	-	BYTES	dtype(object)
TYPE_BF16	kBF16	-	-	-	BF16