❓ [Question] Conversion failure: iresnet

[cjenkins5614]

❓ Question

I was not able to convert my torchscript iresnet model to torch-trt format.

What you have already tried

I was able to convert the resnet101 from torchvision:

model = torchvision.models.resnet101(pretrained=True)
model.eval()
model.to("cuda")
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

And I was able to convert the iresnet101 from https://github.com/iduta/iresnet/blob/master/models/iresnet.py.

But to my own model:

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

WARNING: [Torch-TensorRT TorchScript Conversion Context] - The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. TensorRT maintains only a single logger pointer at any given time, so the existing value, which can be retrieved with getLogger(), will be used instead. In order to use a new logger, first destroy all existing builder, runner or refitter objects.

ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-41-9f8c12fc29e3> in <module>
      7 traced_model = torch.jit.trace(model, data)
      8
----> 9 trt_ts_module = torch_tensorrt.compile(traced_model,
     10     inputs = [data],
     11     # inputs = [torch_tensorrt.Input( # Specify input object with shape and dtype

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/batch_norm.cpp:74] Expected orig_shape.nbDims > 2 to be true but got false
Unable to create batch normalization layer from node: %input.7 : Tensor = aten::batch_norm(%input.5, %self.layer1.0.bn1.weight, %self.layer1.0.bn1.bias, %self.layer1.0.bn1.running_mean, %self.layer1.0.bn1.running_var, %11, %5, %6, %12), scope: __module.layer1/__module.layer1.0/__module.layer1.0.bn1 # /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2381:0

If I used a scripted model instead of a traced one:

scripted_model = torch.jit.script(model, data)

In [4]: trt_ts_module = torch_tensorrt.compile(scripted_model,
   ...:     inputs = [data],
   ...:     enabled_precisions = {torch.float}, # Run with FP16)
   ...: )
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-2a20adf26c71> in <module>
----> 1 trt_ts_module = torch_tensorrt.compile(scripted_model,
      2     inputs = [data],
      3     enabled_precisions = {torch.float}, # Run with FP16)
      4 )

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             # ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:115] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %10834 : Tensor = aten::_convolution(%x.13, %self.layer1.0.downsample.0.weight, %self.conv1.bias, %9502, %9504, %9478, %10832, %10833, %15, %10832, %10832, %10832, %10832)

Environment

Build information about Torch-TensorRT can be found by turning on debug messages

• NGC PyTorch container 21.11-py3

[daeing]

❓ Question

I was not able to convert my torchscript iresnet model to torch-trt format.

What you have already tried

I was able to convert the resnet101 from torchvision:

model = torchvision.models.resnet101(pretrained=True)
model.eval()
model.to("cuda")
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

And I was able to convert the iresnet101 from https://github.com/iduta/iresnet/blob/master/models/iresnet.py.

But to my own model:

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

WARNING: [Torch-TensorRT TorchScript Conversion Context] - The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. TensorRT maintains only a single logger pointer at any given time, so the existing value, which can be retrieved with getLogger(), will be used instead. In order to use a new logger, first destroy all existing builder, runner or refitter objects.

ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-41-9f8c12fc29e3> in <module>
      7 traced_model = torch.jit.trace(model, data)
      8
----> 9 trt_ts_module = torch_tensorrt.compile(traced_model,
     10     inputs = [data],
     11     # inputs = [torch_tensorrt.Input( # Specify input object with shape and dtype

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/batch_norm.cpp:74] Expected orig_shape.nbDims > 2 to be true but got false
Unable to create batch normalization layer from node: %input.7 : Tensor = aten::batch_norm(%input.5, %self.layer1.0.bn1.weight, %self.layer1.0.bn1.bias, %self.layer1.0.bn1.running_mean, %self.layer1.0.bn1.running_var, %11, %5, %6, %12), scope: __module.layer1/__module.layer1.0/__module.layer1.0.bn1 # /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2381:0

If I used a scripted model instead of a traced one:

scripted_model = torch.jit.script(model, data)

In [4]: trt_ts_module = torch_tensorrt.compile(scripted_model,
   ...:     inputs = [data],
   ...:     enabled_precisions = {torch.float}, # Run with FP16)
   ...: )
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-2a20adf26c71> in <module>
----> 1 trt_ts_module = torch_tensorrt.compile(scripted_model,
      2     inputs = [data],
      3     enabled_precisions = {torch.float}, # Run with FP16)
      4 )

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             # ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:115] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %10834 : Tensor = aten::_convolution(%x.13, %self.layer1.0.downsample.0.weight, %self.conv1.bias, %9502, %9504, %9478, %10832, %10833, %15, %10832, %10832, %10832, %10832)

Environment

Build information about Torch-TensorRT can be found by turning on debug messages

• NGC PyTorch container 21.11-py3

I met the same problem, Have you solved it?

[yuezhuang1387]

❓ Question

I was not able to convert my torchscript iresnet model to torch-trt format.

What you have already tried

I was able to convert the resnet101 from torchvision:

model = torchvision.models.resnet101(pretrained=True)
model.eval()
model.to("cuda")
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

And I was able to convert the iresnet101 from https://github.com/iduta/iresnet/blob/master/models/iresnet.py.
But to my own model:

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

WARNING: [Torch-TensorRT TorchScript Conversion Context] - The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. TensorRT maintains only a single logger pointer at any given time, so the existing value, which can be retrieved with getLogger(), will be used instead. In order to use a new logger, first destroy all existing builder, runner or refitter objects.

ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-41-9f8c12fc29e3> in <module>
      7 traced_model = torch.jit.trace(model, data)
      8
----> 9 trt_ts_module = torch_tensorrt.compile(traced_model,
     10     inputs = [data],
     11     # inputs = [torch_tensorrt.Input( # Specify input object with shape and dtype

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/batch_norm.cpp:74] Expected orig_shape.nbDims > 2 to be true but got false
Unable to create batch normalization layer from node: %input.7 : Tensor = aten::batch_norm(%input.5, %self.layer1.0.bn1.weight, %self.layer1.0.bn1.bias, %self.layer1.0.bn1.running_mean, %self.layer1.0.bn1.running_var, %11, %5, %6, %12), scope: __module.layer1/__module.layer1.0/__module.layer1.0.bn1 # /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2381:0

If I used a scripted model instead of a traced one:

scripted_model = torch.jit.script(model, data)

In [4]: trt_ts_module = torch_tensorrt.compile(scripted_model,
   ...:     inputs = [data],
   ...:     enabled_precisions = {torch.float}, # Run with FP16)
   ...: )
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-2a20adf26c71> in <module>
----> 1 trt_ts_module = torch_tensorrt.compile(scripted_model,
      2     inputs = [data],
      3     enabled_precisions = {torch.float}, # Run with FP16)
      4 )

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             # ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:115] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %10834 : Tensor = aten::_convolution(%x.13, %self.layer1.0.downsample.0.weight, %self.conv1.bias, %9502, %9504, %9478, %10832, %10833, %15, %10832, %10832, %10832, %10832)

Environment

Build information about Torch-TensorRT can be found by turning on debug messages

• NGC PyTorch container 21.11-py3

I met the same problem, Have you solved it?

I met the same problem, Have you solved it?

[daeing]

❓ Question

I was not able to convert my torchscript iresnet model to torch-trt format.

What you have already tried

I was able to convert the resnet101 from torchvision:

model = torchvision.models.resnet101(pretrained=True)
model.eval()
model.to("cuda")
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

And I was able to convert the iresnet101 from https://github.com/iduta/iresnet/blob/master/models/iresnet.py.
But to my own model:

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

WARNING: [Torch-TensorRT TorchScript Conversion Context] - The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. TensorRT maintains only a single logger pointer at any given time, so the existing value, which can be retrieved with getLogger(), will be used instead. In order to use a new logger, first destroy all existing builder, runner or refitter objects.

ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-41-9f8c12fc29e3> in <module>
      7 traced_model = torch.jit.trace(model, data)
      8
----> 9 trt_ts_module = torch_tensorrt.compile(traced_model,
     10     inputs = [data],
     11     # inputs = [torch_tensorrt.Input( # Specify input object with shape and dtype

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/batch_norm.cpp:74] Expected orig_shape.nbDims > 2 to be true but got false
Unable to create batch normalization layer from node: %input.7 : Tensor = aten::batch_norm(%input.5, %self.layer1.0.bn1.weight, %self.layer1.0.bn1.bias, %self.layer1.0.bn1.running_mean, %self.layer1.0.bn1.running_var, %11, %5, %6, %12), scope: __module.layer1/__module.layer1.0/__module.layer1.0.bn1 # /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2381:0

If I used a scripted model instead of a traced one:

scripted_model = torch.jit.script(model, data)

In [4]: trt_ts_module = torch_tensorrt.compile(scripted_model,
   ...:     inputs = [data],
   ...:     enabled_precisions = {torch.float}, # Run with FP16)
   ...: )
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-2a20adf26c71> in <module>
----> 1 trt_ts_module = torch_tensorrt.compile(scripted_model,
      2     inputs = [data],
      3     enabled_precisions = {torch.float}, # Run with FP16)
      4 )

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             # ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:115] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %10834 : Tensor = aten::_convolution(%x.13, %self.layer1.0.downsample.0.weight, %self.conv1.bias, %9502, %9504, %9478, %10832, %10833, %15, %10832, %10832, %10832, %10832)

Environment

Build information about Torch-TensorRT can be found by turning on debug messages

• NGC PyTorch container 21.11-py3

I met the same problem, Have you solved it?

I met the same problem, Have you solved it?

it seems the docker's problem. you can try this issue NVIDIA/FasterTransformer#145

[narendasan]

This should be fixed in Torch-TensorRT 1.1.0. We relaxed the dim restriction on batch norm

[HOOLoLo]

Question

I was not able to convert my torchscript iresnet model to torch-trt format.

What you have already tried

I was able to convert the resnet101 from torchvision:

model = torchvision.models.resnet101(pretrained=True)
model.eval()
model.to("cuda")
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

And I was able to convert the iresnet101 from https://github.com/iduta/iresnet/blob/master/models/iresnet.py.

But to my own model:

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

WARNING: [Torch-TensorRT TorchScript Conversion Context] - The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. TensorRT maintains only a single logger pointer at any given time, so the existing value, which can be retrieved with getLogger(), will be used instead. In order to use a new logger, first destroy all existing builder, runner or refitter objects.

ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-41-9f8c12fc29e3> in <module>
      7 traced_model = torch.jit.trace(model, data)
      8
----> 9 trt_ts_module = torch_tensorrt.compile(traced_model,
     10     inputs = [data],
     11     # inputs = [torch_tensorrt.Input( # Specify input object with shape and dtype

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/batch_norm.cpp:74] Expected orig_shape.nbDims > 2 to be true but got false
Unable to create batch normalization layer from node: %input.7 : Tensor = aten::batch_norm(%input.5, %self.layer1.0.bn1.weight, %self.layer1.0.bn1.bias, %self.layer1.0.bn1.running_mean, %self.layer1.0.bn1.running_var, %11, %5, %6, %12), scope: __module.layer1/__module.layer1.0/__module.layer1.0.bn1 # /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2381:0

If I used a scripted model instead of a traced one:

scripted_model = torch.jit.script(model, data)

In [4]: trt_ts_module = torch_tensorrt.compile(scripted_model,
   ...:     inputs = [data],
   ...:     enabled_precisions = {torch.float}, # Run with FP16)
   ...: )
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
ERROR: [Torch-TensorRT TorchScript Conversion Context] - 2: [pointWiseNode.cpp::computeOutputExtents::17] Error Code 2: Internal Error (Assertion nbDims == inputs[i]->extent.nbDims failed.)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-2a20adf26c71> in <module>
----> 1 trt_ts_module = torch_tensorrt.compile(scripted_model,
      2     inputs = [data],
      3     enabled_precisions = {torch.float}, # Run with FP16)
      4 )

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/_compile.py in compile(module, ir, inputs, enabled_precisions, **kwargs)
     95             )
     96             # ts_mod = torch.jit.script(module)
---> 97         return torch_tensorrt.ts.compile(ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs)
     98     elif target_ir == _IRType.fx:
     99         raise RuntimeError("fx is currently not supported")

/opt/conda/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py in compile(module, inputs, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, strict_types, capability, num_min_timing_iters, num_avg_timing_iters, workspace_size, max_batch_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
    117     }
    118
--> 119     compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
    120     compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
    121     return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:115] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %10834 : Tensor = aten::_convolution(%x.13, %self.layer1.0.downsample.0.weight, %self.conv1.bias, %9502, %9504, %9478, %10832, %10833, %15, %10832, %10832, %10832, %10832)

Environment

Build information about Torch-TensorRT can be found by turning on debug messages

• NGC PyTorch container 21.11-py3

This should be fixed in Torch-TensorRT 1.1.0. We relaxed the dim restriction on batch norm

I used the 21.12 docker image (https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel_21-12.html#rel_21-12) which's Torch-TensorRT version is 1.1.0.0.a.0 but got the same problem,

[HOOLoLo]

Has this bug been solved? @narendasan

[narendasan]

Yes, just tried the provided code in 1.12.0a0 and it runs fine. Here is the source of what I ran.

import torch
from torch import nn


def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=3,
                     stride=stride,
                     padding=dilation,
                     groups=groups,
                     bias=False,
                     dilation=dilation)


def conv1x1(in_planes, out_planes, stride=1):
    """1x1 convolution"""
    return nn.Conv2d(in_planes,
                     out_planes,
                     kernel_size=1,
                     stride=stride,
                     bias=False)


class IBasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride=1, downsample=None,
                 groups=1, base_width=64, dilation=1):
        super(IBasicBlock, self).__init__()
        if groups != 1 or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,)
        self.conv1 = conv3x3(inplanes, planes)
        self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.prelu = nn.PReLU(planes)
        self.conv2 = conv3x3(planes, planes, stride)
        self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x
        out = self.bn1(x)
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.prelu(out)
        out = self.conv2(out)
        out = self.bn3(out)
        if self.downsample is not None:
            identity = self.downsample(x)
        out += identity
        return out


class IResNet(nn.Module):
    fc_scale = 7 * 7
    def __init__(self,
                 block, layers, dropout=0, num_features=512, zero_init_residual=False,
                 groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False):
        super(IResNet, self).__init__()
        self.fp16 = fp16
        self.inplanes = 64
        self.dilation = 1
        if replace_stride_with_dilation is None:
            replace_stride_with_dilation = [False, False, False]
        if len(replace_stride_with_dilation) != 3:
            raise ValueError("replace_stride_with_dilation should be None "
                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
        self.groups = groups
        self.base_width = width_per_group
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05)
        self.prelu = nn.PReLU(self.inplanes)
        self.layer1 = self._make_layer(block, 64, layers[0], stride=2)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[0])
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[1])
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       dilate=replace_stride_with_dilation[2])
        self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,)
        self.dropout = nn.Dropout(p=dropout, inplace=True)
        self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features)
        self.features = nn.BatchNorm1d(num_features, eps=1e-05)
        nn.init.constant_(self.features.weight, 1.0)
        self.features.weight.requires_grad = False

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, 0, 0.1)
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, IBasicBlock):
                    nn.init.constant_(m.bn2.weight, 0)

    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
        downsample = None
        previous_dilation = self.dilation
        if dilate:
            self.dilation *= stride
            stride = 1
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ),
            )
        layers = []
        layers.append(
            block(self.inplanes, planes, stride, downsample, self.groups,
                  self.base_width, previous_dilation))
        self.inplanes = planes * block.expansion
        for _ in range(1, blocks):
            layers.append(
                block(self.inplanes,
                      planes,
                      groups=self.groups,
                      base_width=self.base_width,
                      dilation=self.dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.prelu(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.bn2(x)
        x = torch.flatten(x, 1)
        x = self.dropout(x)
        x = self.fc(x.float() if self.fp16 else x)
        x = self.features(x)
        return x


def _iresnet(arch, block, layers, pretrained, progress, **kwargs):
    model = IResNet(block, layers, **kwargs)
    if pretrained:
        raise ValueError()
    return model


def iresnet(pretrained=False, progress=True, **kwargs):
    return _iresnet('iresnet', IBasicBlock, [3, 13, 30, 3], pretrained,
                    progress, **kwargs)

import torch_tensorrt

model = iresnet().to(device="cuda")
model.eval()
data = torch.zeros((1, 3, 112, 112)).to("cuda")
traced_model = torch.jit.trace(model, data)

trt_ts_module = torch_tensorrt.compile(traced_model,
    inputs = [data],
    enabled_precisions = {torch.float}, # Run with FP16)
)

print(trt_ts_module(data))

Just so that we can tell what has been addressed and what hasn't, I am going to close this but feel free to reopen if there is still issues.

[hamjam]

I have similar problem when trying to adopt Citrinet-Example.ipynb.
My model is a fine-tuned Citrinet512 from NeMo.
By the way my pytorch_tesorrt version is 1.3.0.
Here is my code:

import torch
import torch.nn as nn
import torch_tensorrt as torchtrt
import argparse

precisions = [torch.float, torch.half]
batch_sizes = [1,8,32,128]
model = torch.jit.load(model_path+f"{variant}.ts")
model.eval()
for precision in precisions:
    for batch_size in batch_sizes:
        compile_settings = {
            "inputs": [torchtrt.Input(shape=[batch_size, 80, 1488],  dtype=torch.int), torchtrt.Input(shape=[1, batch_size],  dtype=torch.int)],
            "enabled_precisions": {precision},
            "workspace_size": 2000000000,
            "truncate_long_and_double": True,
        }
        print(f"Generating Torchscript-TensorRT module for batchsize {batch_size} precision {precision}")
        trt_ts_module = torchtrt.compile(model, **compile_settings)
        torch.jit.save(trt_ts_module, f"{variant}_bs{batch_size}_{precision}.torch-tensorrt")

Output:

RuntimeError                              Traceback (most recent call last)
Input In [6], in <module>
    14 compile_settings = {
    15     "inputs": [torchtrt.Input(shape=[batch_size, 80, 1488],  dtype=torch.int), torchtrt.Input(shape=[1, batch_size],  dtype=torch.int)],
    16     "enabled_precisions": {precision},
    17     "workspace_size": 2000000000,
    18     "truncate_long_and_double": True,
    19 }
    20 print(f"Generating Torchscript-TensorRT module for batchsize {batch_size} precision {precision}")
---> 21 trt_ts_module = torchtrt.compile(model, **compile_settings)
    22 torch.jit.save(trt_ts_module, f"{variant}_bs{batch_size}_{precision}.torch-tensorrt")

File ~/nvidia-nemo-asr-training/venv/lib/python3.9/site-packages/torch_tensorrt/_compile.py:125, in compile(module, ir, inputs, enabled_precisions, **kwargs)
   120         logging.log(
   121             logging.Level.Info,
   122             "Module was provided as a torch.nn.Module, trying to script the module with torch.jit.script. In the event of a failure please preconvert your module to TorchScript",
   123         )
   124         ts_mod = torch.jit.script(module)
--> 125     return torch_tensorrt.ts.compile(
   126         ts_mod, inputs=inputs, enabled_precisions=enabled_precisions, **kwargs
   127     )
   128 elif target_ir == _IRType.fx:
   129     if (
   130         torch.float16 in enabled_precisions
   131         or torch_tensorrt.dtype.half in enabled_precisions
   132     ):

File ~/nvidia-nemo-asr-training/venv/lib/python3.9/site-packages/torch_tensorrt/ts/_compiler.py:136, in compile(module, inputs, input_signature, device, disable_tf32, sparse_weights, enabled_precisions, refit, debug, capability, num_avg_timing_iters, workspace_size, dla_sram_size, dla_local_dram_size, dla_global_dram_size, calibrator, truncate_long_and_double, require_full_compilation, min_block_size, torch_executed_ops, torch_executed_modules)
   110     raise ValueError(
   111         f"require_full_compilation is enabled however the list of modules and ops to run in torch is not empty. Found: torch_executed_ops: {torch_executed_ops}, torch_executed_modules: {torch_executed_modules}"
   112     )
   114 spec = {
   115     "inputs": inputs,
   116     "input_signature": input_signature,
  (...)
   133     },
   134 }
--> 136 compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
   137 compiled_module = torch.jit._recursive.wrap_cpp_module(compiled_cpp_mod)
   138 return compiled_module

RuntimeError: [Error thrown at core/conversion/converters/impl/conv_deconv.cpp:129] Expected orig_dims.nbDims > 2 to be true but got false
Unable to create convolution layer from node: %18096 : Tensor = aten::_convolution(%18093, %33, %32, %34, %35, %34, %18094, %18095, %30, %18094, %18094, %18094, %18094)