how to check precision of each layer after quantize to INT8 model

[minhhb9x]

Description

Hi.
After converting onnx model to INT8 engine model, I want to check precision of each layer. Can sb give me an example, please? Thank you very much.

Environment

TensorRT Version:

NVIDIA GPU:
RTX 3050 Laptop gpu
NVIDIA Driver Version:

CUDA Version:
12.1
CUDNN Version:

Operating System:
Window 10
Python Version (if applicable):
3.10
Tensorflow Version (if applicable):

PyTorch Version (if applicable):

Baremetal or Container (if so, version):

[bernardrb]

Using the python API, each layer has a precision attribute. Before creating the engine, you can set, and check the precision.

Furthermore, you can inspect the model.engine. This is the script I use:

def main(args):    
    trt_logger = trt.Logger(trt.Logger.VERBOSE)

    # Load the engine
    with open(args.engine, "rb") as f, trt.Runtime(trt_logger) as runtime:
        engine = runtime.deserialize_cuda_engine(f.read())

    # Create an inspector
    inspector = engine.create_engine_inspector()

    # Print the layer information
    for i in range(engine.num_bindings):
        print(inspector.get_layer_information(i, trt.LayerInformationFormat.JSON))

    # Print the engine information
    print(inspector.get_engine_information(trt.LayerInformationFormat.JSON))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Inspect a TensorRT engine")
    parser.add_argument("--engine", help="Path to the engine file", required=True)
    args = parser.parse_args()
    main(args)

Hope this helps!

[minhhb9x]

thank you. I will try

[minhhb9x]

@bernardrb I only got the name of the layers:

"/model.0/conv/Conv + /model.0/act/Relu"

"Reformatting CopyNode for Input Tensor 0 to /model.1/conv/Conv + /model.1/act/Relu"

{"Layers": ["/model.0/conv/Conv + /model.0/act/Relu"
,"Reformatting CopyNode for Input Tensor 0 to /model.1/conv/Conv + /model.1/act/Relu"
,"/model.1/conv/Conv + /model.1/act/Relu"
,"/model.2/cv1/conv/Conv + /model.2/cv1/act/Relu"
,"/model.2/m.0/cv1/conv/Conv + /model.2/m.0/cv1/act/Relu"
,"/model.2/m.0/cv2/conv/Conv + /model.2/m.0/cv2/act/Relu"
,"PWN(/model.2/m.0/Add)"
,"/model.2/Split_output_0 copy"
,"/model.2/Split_output_1 copy"
,"/model.2/cv2/conv/Conv + /model.2/cv2/act/Relu"
,"/model.3/conv/Conv + /model.3/act/Relu"
,"/model.4/cv1/conv/Conv + /model.4/cv1/act/Relu"
,"/model.4/Split_2"
,"/model.4/m.0/cv1/conv/Conv + /model.4/m.0/cv1/act/Relu"
,"/model.4/m.0/cv2/conv/Conv + /model.4/m.0/cv2/act/Relu + PWN(/model.4/m.0/Add)"
,"/model.4/m.1/cv1/conv/Conv + /model.4/m.1/cv1/act/Relu"
,"/model.4/m.1/cv2/conv/Conv + /model.4/m.1/cv2/act/Relu + PWN(/model.4/m.1/Add)"
,"/model.4/Split_output_0 copy"
,"/model.4/Split_output_1 copy"
,"/model.4/m.0/Add_output_0 copy"
,"/model.4/m.1/Add_output_0 copy"
,"/model.4/cv2/conv/Conv + /model.4/cv2/act/Relu"
,"/model.5/conv/Conv + /model.5/act/Relu"
,"/model.6/cv1/conv/Conv + /model.6/cv1/act/Relu"
,"/model.6/Split_4"
,"/model.6/m.0/cv1/conv/Conv + /model.6/m.0/cv1/act/Relu"
,"/model.6/m.0/cv2/conv/Conv + /model.6/m.0/cv2/act/Relu + PWN(/model.6/m.0/Add)"
,"/model.6/m.1/cv1/conv/Conv + /model.6/m.1/cv1/act/Relu"
,"/model.6/m.1/cv2/conv/Conv + /model.6/m.1/cv2/act/Relu + PWN(/model.6/m.1/Add)"
,"/model.6/Split_output_0 copy"
,"/model.6/Split_output_1 copy"
,"/model.6/m.0/Add_output_0 copy"
,"/model.6/m.1/Add_output_0 copy"
,"/model.6/cv2/conv/Conv + /model.6/cv2/act/Relu"
,"/model.7/conv/Conv + /model.7/act/Relu"
,"/model.8/cv1/conv/Conv + /model.8/cv1/act/Relu"
,"/model.8/Split_6"
,"/model.8/m.0/cv1/conv/Conv + /model.8/m.0/cv1/act/Relu"
,"/model.8/m.0/cv2/conv/Conv + /model.8/m.0/cv2/act/Relu + PWN(/model.8/m.0/Add)"
,"/model.8/Split_output_0 copy"
,"/model.8/Split_output_1 copy"
,"/model.8/m.0/Add_output_0 copy"
,"/model.8/cv2/conv/Conv + /model.8/cv2/act/Relu"
,"/model.9/cv1/conv/Conv + /model.9/cv1/act/Relu"
,"/model.9/m/MaxPool"
,"/model.9/m_1/MaxPool"
,"/model.9/m_2/MaxPool"
,"/model.9/cv1/act/Relu_output_0 copy"
,"/model.9/m/MaxPool_output_0 copy"
,"/model.9/m_1/MaxPool_output_0 copy"
,"/model.9/cv2/conv/Conv + /model.9/cv2/act/Relu"
,"/model.10/Resize"
,"/model.10/Resize_output_0 copy"
,"/model.12/cv1/conv/Conv + /model.12/cv1/act/Relu"
,"/model.12/m.0/cv1/conv/Conv + /model.12/m.0/cv1/act/Relu"
,"/model.12/m.0/cv2/conv/Conv + /model.12/m.0/cv2/act/Relu"
,"/model.12/Split_output_0 copy"
,"/model.12/Split_output_1 copy"
,"/model.12/cv2/conv/Conv + /model.12/cv2/act/Relu"
,"/model.13/Resize"
,"/model.13/Resize_output_0 copy"
,"/model.15/cv1/conv/Conv + /model.15/cv1/act/Relu"
,"/model.15/m.0/cv1/conv/Conv + /model.15/m.0/cv1/act/Relu"
,"/model.15/m.0/cv2/conv/Conv + /model.15/m.0/cv2/act/Relu"
,"/model.15/Split_output_0 copy"
,"/model.15/Split_output_1 copy"
,"/model.15/cv2/conv/Conv + /model.15/cv2/act/Relu"
,"/model.16/conv/Conv + /model.16/act/Relu"
,"/model.12/cv2/act/Relu_output_0 copy"
,"/model.18/cv1/conv/Conv + /model.18/cv1/act/Relu"
,"/model.18/m.0/cv1/conv/Conv + /model.18/m.0/cv1/act/Relu"
,"/model.18/m.0/cv2/conv/Conv + /model.18/m.0/cv2/act/Relu"
,"/model.18/Split_output_0 copy"
,"/model.18/Split_output_1 copy"
,"/model.18/cv2/conv/Conv + /model.18/cv2/act/Relu"
,"/model.19/conv/Conv + /model.19/act/Relu"
,"/model.9/cv2/act/Relu_output_0 copy"
,"/model.21/cv1/conv/Conv + /model.21/cv1/act/Relu"
,"/model.21/m.0/cv1/conv/Conv + /model.21/m.0/cv1/act/Relu"
,"/model.21/m.0/cv2/conv/Conv + /model.21/m.0/cv2/act/Relu"
,"/model.21/Split_output_0 copy"
,"/model.21/Split_output_1 copy"
,"/model.21/cv2/conv/Conv + /model.21/cv2/act/Relu"
,"/model.22/cv2.2/cv2.2.0/conv/Conv + /model.22/cv2.2/cv2.2.0/act/Relu || /model.22/cv3.2/cv3.2.0/conv/Conv + /model.22/cv3.2/cv3.2.0/act/Relu"
,"/model.22/cv2.2/cv2.2.1/conv/Conv + /model.22/cv2.2/cv2.2.1/act/Relu"
,"/model.22/cv2.2/cv2.2.2/Conv"
,"/model.22/cv3.2/cv3.2.1/conv/Conv + /model.22/cv3.2/cv3.2.1/act/Relu"
,"/model.22/cv3.2/cv3.2.2/Conv"
,"/model.22/Reshape_2"
,"/model.22/Reshape_2_copy_output"
,"/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu"
,"/model.22/cv2.1/cv2.1.1/conv/Conv + /model.22/cv2.1/cv2.1.1/act/Relu"
,"/model.22/cv2.1/cv2.1.2/Conv"
,"/model.22/cv3.1/cv3.1.1/conv/Conv + /model.22/cv3.1/cv3.1.1/act/Relu"
,"/model.22/cv3.1/cv3.1.2/Conv"
,"/model.22/Reshape_1"
,"/model.22/Reshape_1_copy_output"
,"/model.22/cv2.0/cv2.0.0/conv/Conv + /model.22/cv2.0/cv2.0.0/act/Relu || /model.22/cv3.0/cv3.0.0/conv/Conv + /model.22/cv3.0/cv3.0.0/act/Relu"
,"/model.22/cv2.0/cv2.0.1/conv/Conv + /model.22/cv2.0/cv2.0.1/act/Relu"
,"/model.22/cv2.0/cv2.0.2/Conv"
,"/model.22/cv3.0/cv3.0.1/conv/Conv + /model.22/cv3.0/cv3.0.1/act/Relu"
,"/model.22/cv3.0/cv3.0.2/Conv"
,"/model.22/Reshape"
,"/model.22/Reshape_copy_output"
,"/model.22/dfl/Reshape + /model.22/dfl/Transpose"
,"/model.22/dfl/Softmax"
,"/model.22/dfl/conv/Conv"
,"/model.22/dfl/Reshape_1"
,"PWN(/model.22/Sigmoid)"
,"/model.22/Constant_12_output_0 + (Unnamed Layer* 242) [Shuffle]"
,"/model.22/Constant_10_output_0"
,"PWN(/model.22/Add_1)"
,"/model.22/Constant_9_output_0"
,"/model.22/Sub"
,"/model.22/Sub_1"
,"PWN(/model.22/Add_2, PWN(/model.22/Constant_11_output_0 + (Unnamed Layer* 237) [Shuffle], /model.22/Div_1))"
,"/model.22/Div_1_output_0 copy"
,"/model.22/Mul_2"
,"/model.22/Mul_2_output_0 copy"
],
"Bindings": ["images"
,"output0"
]}

[bernardrb]

From https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#engine-inspector,

Note that the level of detail in the engine/layer information depends on the ProfilingVerbosity builder config setting when the engine is built. By default, ProfilingVerbosity is set to kLAYER_NAMES_ONLY, so only the layer names will be printed. If ProfilingVerbosity is set to kNONE, then no information will be printed; if it is set to kDETAILED, then detailed information will be printed.

If you are building your engine in a script, it might look like this:

self.config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED

engine_bytes = None
try:
    engine_bytes = self.builder.build_serialized_network(
        self.network, self.config)

From https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#trtexec-flags,

If you are using trtexec, set the verbose flag when building the engine. Something like this,

trtexec --verbose model.onnx

[minhhb9x]

Ok now I can see the datatype of my layers. However, some layer's datatypes are still FP32. Do you know why?

...
{
  "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.18/cv2/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,128,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,144,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "ParameterType": "Convolution",
  "Kernel": [3,3],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [1,1],
  "PostPadding": [1,1],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 144,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 165888},
  "Bias": {"Type": "Float", "Count": 144},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "RELU",
  "HasBias": 1,
  "HasReLU": 1,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8i8_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_vect_c_32_tilesize128x96x64_stage3_warpsize4x1x1_g1_tensor16x8x32_t1r3s3",        
  "TacticValue": "0xd277f13d771603ee",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.0/conv/Conv]\u001e[ONNX Layer: /model.22/cv2.1/cv2.1.0/act/Relu]\u001e[ONNX Layer: /model.22/cv3.1/cv3.1.0/conv/Conv]\u001e[ONNX Layer: /model.22/cv3.1/cv3.1.0/act/Relu]"
},{
  "Name": "/model.22/cv2.1/cv2.1.1/conv/Conv + /model.22/cv2.1/cv2.1.1/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "ParameterType": "Convolution",
  "Kernel": [3,3],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [1,1],
  "PostPadding": [1,1],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 64,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 36864},
  "Bias": {"Type": "Float", "Count": 64},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "RELU",
  "HasBias": 1,
  "HasReLU": 1,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8i8_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_vect_c_32_tilesize128x64x64_stage6_warpsize2x2x1_g1_tensor16x8x32_t1r3s3",        
  "TacticValue": "0x705baf38e41eee0b",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.1/conv/Conv]\u001e[ONNX Layer: /model.22/cv2.1/cv2.1.1/act/Relu]"
},{
  "Name": "/model.22/cv2.1/cv2.1.2/Conv",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/Concat_1_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Row major linear FP32"
  }],
  "ParameterType": "Convolution",
  "Kernel": [1,1],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [0,0],
  "PostPadding": [0,0],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 64,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 4096},
  "Bias": {"Type": "Float", "Count": 64},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "NONE",
  "HasBias": 1,
  "HasReLU": 0,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8f32_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_tilesize128x32x64_stage4_warpsize4x1x1_g1_tensor16x8x32_t1r1s1_alignc4",
  "TacticValue": "0x733ba2a91a48d431",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.2/Conv]"
},{
  "Name": "/model.22/cv3.1/cv3.1.1/conv/Conv + /model.22/cv3.1/cv3.1.1/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,80,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv3.1/cv3.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,80,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
...

I'm using python to convert onnx to INT8 engine. And this is my code:

import argparse
import os
import json
import tensorrt as trt
from datetime import datetime
import onnx
import calibration

TRT_LOGGER = trt.Logger()

def parse_args():
    parser = argparse.ArgumentParser(description='Convert ONNX models to TensorRT')
        
    # Sample image
    parser.add_argument('--batch_size', type=int, help='data batch size',
        default=1)
    parser.add_argument('--img_size', help='input size',
        default=[3, 640, 640])

    # Model path
    parser.add_argument('--onnx_model_path',  help='onnx model path',
        default='./onnx_model.onnx')
    parser.add_argument('--tensorrt_engine_path',  help='tensorrt engine path',
        default='./yolov5s_640_384_pfg_dynamic_max_batchsize_8_FP16.engine')

    # TensorRT engine params
    parser.add_argument('--dynamic_axes', help='dynamic batch input or output',
        default='True')
    parser.add_argument('--engine_precision', help='precision of TensorRT engine', choices=['FP32', 'FP16', 'INT8'], 
        default='INT8')
    parser.add_argument('--min_engine_batch_size', type=int, help='set the min input data size of model for inference', 
        default=1)
    parser.add_argument('--opt_engine_batch_size', type=int, help='set the most used input data size of model for inference', 
        default=1)
    parser.add_argument('--max_engine_batch_size', type=int, help='set the max input data size of model for inference', 
        default=1)
    parser.add_argument('--engine_workspace', type=int, help='workspace of engine', 
        default=1024)
    args = string_to_bool(parser.parse_args())

    return args

def extract_metadata(onnx_model_path):
    # Load ONNX model
    model_onnx = onnx.load(onnx_model_path)

    # Extract metadata
    metadata = {}
    for prop in model_onnx.metadata_props:
        metadata[prop.key] = prop.value
    return metadata


def string_to_bool(args):
    if args.dynamic_axes.lower() in ('true'): args.dynamic_axes = True
    else: args.dynamic_axes = False
    return args


def build_engine(onnx_model_path, tensorrt_engine_path, engine_precision, dynamic_axes, \
    img_size, batch_size, min_engine_batch_size, opt_engine_batch_size, max_engine_batch_size):
    
    metadata = extract_metadata(onnx_model_path)
    print(metadata)
    # Builder
    logger = trt.Logger(trt.Logger.ERROR)
    builder = trt.Builder(logger)
    network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)

    if engine_precision == "INT8":
        print('PTQ enabled!')
        network_flags = network_flags | (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_PRECISION))

    network = builder.create_network(network_flags)

    profile = builder.create_optimization_profile()

    config = builder.create_builder_config()
    config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED

    # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 3 << 30)
    # Set FP16 
    if engine_precision == 'FP16':
        config.set_flag(trt.BuilderFlag.FP16)
    elif engine_precision == 'INT8':
        config.flags |= 1 << int(trt.BuilderFlag.INT8)
        config.flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS)
        calib_loader = calibration.DataLoader(batch_size, 128, 'coco128/images/train2017', 640, 640)
        config.int8_calibrator = calibration.Calibrator(calib_loader, './coco128calib.cache')
    
    # Onnx parser
    parser = trt.OnnxParser(network, logger)

    if not os.path.exists(onnx_model_path):
        print("Failed finding ONNX file!")
        exit()
    print("Succeeded finding ONNX file!")
    with open(onnx_model_path, "rb") as model:
        if not parser.parse(model.read()):
            print("Failed parsing .onnx file!")
            for error in range(parser.num_errors):
                print(parser.get_error(error))
            exit()
        print("Succeeded parsing .onnx file!")
        
    # Input
    inputTensor = network.get_input(0) 
    # Dynamic batch (min, opt, max)
    print('inputTensor.name:', inputTensor.name)
    if dynamic_axes:
        profile.set_shape(inputTensor.name, (min_engine_batch_size, img_size[0], img_size[1], img_size[2]), \
            (opt_engine_batch_size, img_size[0], img_size[1], img_size[2]), \
            (max_engine_batch_size, img_size[0], img_size[1], img_size[2]))
        print('Set dynamic')
    else:
        profile.set_shape(inputTensor.name, (batch_size, img_size[0], img_size[1], img_size[2]), \
            (batch_size, img_size[0], img_size[1], img_size[2]), \
            (batch_size, img_size[0], img_size[1], img_size[2]))
    config.add_optimization_profile(profile)
    #network.unmark_output(network.get_output(0))
    
    # Write engine
    engineString = builder.build_serialized_network(network, config)
    if engineString == None:
        print("Failed building engine!")
        exit()
    print("Succeeded building engine!")

    # Chuyển từ dictionary sang JSON và encode nó
    metaString = json.dumps(metadata).encode('utf-8')
    
    # Lưu engine cùng với metadata vào file
    with open(tensorrt_engine_path, "wb") as f:
        # Ghi độ dài của metadata
        f.write(len(metaString).to_bytes(4, byteorder='little'))
        # Ghi metadata
        f.write(metaString)
        # Ghi engine
        f.write(engineString)
        


def main():
    args = parse_args()    
    # Build TensorRT engine
    build_engine(args.onnx_model_path, args.tensorrt_engine_path, args.engine_precision, args.dynamic_axes, \
        args.img_size, args.batch_size, args.min_engine_batch_size, args.opt_engine_batch_size, \
        args.max_engine_batch_size)
    
   
    
if __name__ == '__main__': 
    main()

[bernardrb]

config.flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS)

If you want all layers to use INT8 change it to,

config.flags |= 1 << int(trt.BuilderFlag.OBEY_PRECISION_CONSTRAINTS)

Note, however, that this can make the build fail as some layers can require FP32 to work.

[zerollzeng]

To see layer precision you can also check verbose log, from Engine Information section.

[minhhb9x]

Thank you. I get it.

[braindevices]

AttributeError: 'tensorrt_bindings.tensorrt.ICudaEngine' object has no attribute 'num_bindings'

there is no num_bindings at all

[ys198918]

Ok now I can see the datatype of my layers. However, some layer's datatypes are still FP32. Do you know why?

...
{
  "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.18/cv2/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,128,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,144,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "ParameterType": "Convolution",
  "Kernel": [3,3],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [1,1],
  "PostPadding": [1,1],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 144,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 165888},
  "Bias": {"Type": "Float", "Count": 144},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "RELU",
  "HasBias": 1,
  "HasReLU": 1,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8i8_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_vect_c_32_tilesize128x96x64_stage3_warpsize4x1x1_g1_tensor16x8x32_t1r3s3",        
  "TacticValue": "0xd277f13d771603ee",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.0/conv/Conv]\u001e[ONNX Layer: /model.22/cv2.1/cv2.1.0/act/Relu]\u001e[ONNX Layer: /model.22/cv3.1/cv3.1.0/conv/Conv]\u001e[ONNX Layer: /model.22/cv3.1/cv3.1.0/act/Relu]"
},{
  "Name": "/model.22/cv2.1/cv2.1.1/conv/Conv + /model.22/cv2.1/cv2.1.1/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "ParameterType": "Convolution",
  "Kernel": [3,3],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [1,1],
  "PostPadding": [1,1],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 64,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 36864},
  "Bias": {"Type": "Float", "Count": 64},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "RELU",
  "HasBias": 1,
  "HasReLU": 1,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8i8_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_vect_c_32_tilesize128x64x64_stage6_warpsize2x2x1_g1_tensor16x8x32_t1r3s3",        
  "TacticValue": "0x705baf38e41eee0b",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.1/conv/Conv]\u001e[ONNX Layer: /model.22/cv2.1/cv2.1.1/act/Relu]"
},{
  "Name": "/model.22/cv2.1/cv2.1.2/Conv",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/Concat_1_output_0",
    "Location": "Device",
    "Dimensions": [1,64,40,40],
    "Format/Datatype": "Row major linear FP32"
  }],
  "ParameterType": "Convolution",
  "Kernel": [1,1],
  "PaddingMode": "kEXPLICIT_ROUND_DOWN",
  "PrePadding": [0,0],
  "PostPadding": [0,0],
  "Stride": [1,1],
  "Dilation": [1,1],
  "OutMaps": 64,
  "Groups": 1,
  "Weights": {"Type": "Int8", "Count": 4096},
  "Bias": {"Type": "Float", "Count": 64},
  "HasSparseWeights": 0,
  "HasDynamicFilter": 0,
  "HasDynamicBias": 0,
  "HasResidual": 0,
  "ConvXAsActInputIdx": -1,
  "BiasAsActInputIdx": -1,
  "ResAsActInputIdx": -1,
  "Activation": "NONE",
  "HasBias": 1,
  "HasReLU": 0,
  "TacticName": "sm80_xmma_fprop_implicit_gemm_interleaved_i8f32_i8i32_f32_nchw_vect_c_32kcrs_vect_c_32_nchw_tilesize128x32x64_stage4_warpsize4x1x1_g1_tensor16x8x32_t1r1s1_alignc4",
  "TacticValue": "0x733ba2a91a48d431",
  "StreamId": 0,
  "Metadata": "[ONNX Layer: /model.22/cv2.1/cv2.1.2/Conv]"
},{
  "Name": "/model.22/cv3.1/cv3.1.1/conv/Conv + /model.22/cv3.1/cv3.1.1/act/Relu",
  "LayerType": "CaskConvolution",
  "Inputs": [
  {
    "Name": "/model.22/cv2.1/cv2.1.0/conv/Conv + /model.22/cv2.1/cv2.1.0/act/Relu || /model.22/cv3.1/cv3.1.0/conv/Conv + /model.22/cv3.1/cv3.1.0/act/Relu",
    "Location": "Device",
    "Dimensions": [1,80,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
  "Outputs": [
  {
    "Name": "/model.22/cv3.1/cv3.1.1/act/Relu_output_0",
    "Location": "Device",
    "Dimensions": [1,80,40,40],
    "Format/Datatype": "Thirty-two wide channel vectorized row major Int8 format"
  }],
...

I'm using python to convert onnx to INT8 engine. And this is my code:

import argparse
import os
import json
import tensorrt as trt
from datetime import datetime
import onnx
import calibration

TRT_LOGGER = trt.Logger()

def parse_args():
    parser = argparse.ArgumentParser(description='Convert ONNX models to TensorRT')
        
    # Sample image
    parser.add_argument('--batch_size', type=int, help='data batch size',
        default=1)
    parser.add_argument('--img_size', help='input size',
        default=[3, 640, 640])

    # Model path
    parser.add_argument('--onnx_model_path',  help='onnx model path',
        default='./onnx_model.onnx')
    parser.add_argument('--tensorrt_engine_path',  help='tensorrt engine path',
        default='./yolov5s_640_384_pfg_dynamic_max_batchsize_8_FP16.engine')

    # TensorRT engine params
    parser.add_argument('--dynamic_axes', help='dynamic batch input or output',
        default='True')
    parser.add_argument('--engine_precision', help='precision of TensorRT engine', choices=['FP32', 'FP16', 'INT8'], 
        default='INT8')
    parser.add_argument('--min_engine_batch_size', type=int, help='set the min input data size of model for inference', 
        default=1)
    parser.add_argument('--opt_engine_batch_size', type=int, help='set the most used input data size of model for inference', 
        default=1)
    parser.add_argument('--max_engine_batch_size', type=int, help='set the max input data size of model for inference', 
        default=1)
    parser.add_argument('--engine_workspace', type=int, help='workspace of engine', 
        default=1024)
    args = string_to_bool(parser.parse_args())

    return args

def extract_metadata(onnx_model_path):
    # Load ONNX model
    model_onnx = onnx.load(onnx_model_path)

    # Extract metadata
    metadata = {}
    for prop in model_onnx.metadata_props:
        metadata[prop.key] = prop.value
    return metadata


def string_to_bool(args):
    if args.dynamic_axes.lower() in ('true'): args.dynamic_axes = True
    else: args.dynamic_axes = False
    return args


def build_engine(onnx_model_path, tensorrt_engine_path, engine_precision, dynamic_axes, \
    img_size, batch_size, min_engine_batch_size, opt_engine_batch_size, max_engine_batch_size):
    
    metadata = extract_metadata(onnx_model_path)
    print(metadata)
    # Builder
    logger = trt.Logger(trt.Logger.ERROR)
    builder = trt.Builder(logger)
    network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)

    if engine_precision == "INT8":
        print('PTQ enabled!')
        network_flags = network_flags | (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_PRECISION))

    network = builder.create_network(network_flags)

    profile = builder.create_optimization_profile()

    config = builder.create_builder_config()
    config.profiling_verbosity = trt.ProfilingVerbosity.DETAILED

    # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 3 << 30)
    # Set FP16 
    if engine_precision == 'FP16':
        config.set_flag(trt.BuilderFlag.FP16)
    elif engine_precision == 'INT8':
        config.flags |= 1 << int(trt.BuilderFlag.INT8)
        config.flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS)
        calib_loader = calibration.DataLoader(batch_size, 128, 'coco128/images/train2017', 640, 640)
        config.int8_calibrator = calibration.Calibrator(calib_loader, './coco128calib.cache')
    
    # Onnx parser
    parser = trt.OnnxParser(network, logger)

    if not os.path.exists(onnx_model_path):
        print("Failed finding ONNX file!")
        exit()
    print("Succeeded finding ONNX file!")
    with open(onnx_model_path, "rb") as model:
        if not parser.parse(model.read()):
            print("Failed parsing .onnx file!")
            for error in range(parser.num_errors):
                print(parser.get_error(error))
            exit()
        print("Succeeded parsing .onnx file!")
        
    # Input
    inputTensor = network.get_input(0) 
    # Dynamic batch (min, opt, max)
    print('inputTensor.name:', inputTensor.name)
    if dynamic_axes:
        profile.set_shape(inputTensor.name, (min_engine_batch_size, img_size[0], img_size[1], img_size[2]), \
            (opt_engine_batch_size, img_size[0], img_size[1], img_size[2]), \
            (max_engine_batch_size, img_size[0], img_size[1], img_size[2]))
        print('Set dynamic')
    else:
        profile.set_shape(inputTensor.name, (batch_size, img_size[0], img_size[1], img_size[2]), \
            (batch_size, img_size[0], img_size[1], img_size[2]), \
            (batch_size, img_size[0], img_size[1], img_size[2]))
    config.add_optimization_profile(profile)
    #network.unmark_output(network.get_output(0))
    
    # Write engine
    engineString = builder.build_serialized_network(network, config)
    if engineString == None:
        print("Failed building engine!")
        exit()
    print("Succeeded building engine!")

    # Chuyển từ dictionary sang JSON và encode nó
    metaString = json.dumps(metadata).encode('utf-8')
    
    # Lưu engine cùng với metadata vào file
    with open(tensorrt_engine_path, "wb") as f:
        # Ghi độ dài của metadata
        f.write(len(metaString).to_bytes(4, byteorder='little'))
        # Ghi metadata
        f.write(metaString)
        # Ghi engine
        f.write(engineString)
        


def main():
    args = parse_args()    
    # Build TensorRT engine
    build_engine(args.onnx_model_path, args.tensorrt_engine_path, args.engine_precision, args.dynamic_axes, \
        args.img_size, args.batch_size, args.min_engine_batch_size, args.opt_engine_batch_size, \
        args.max_engine_batch_size)
    
   
    
if __name__ == '__main__': 
    main()

how did you get this @minhhb9x