onnx to haf error

General
1

I used mmdet’s resnet+fpn as the backbone and neck to convert to onnx, but failed to convert to haf. I tried some methods but failed.
Convert code

import torch
import torch.nn as nn
from mmcv import Config
from mmdet3d.models import build_model
import os
from projects.mmdet3d_plugin.models.sparse4d import Sparse4D

class BackboneNeckModel(nn.Module):
    def __init__(self, model):
        super(BackboneNeckModel, self).__init__()
        # 只包含 backbone 和 neck
        self.backbone = model.img_backbone
        self.neck = model.img_neck
        
    def forward(self, x):
        # 调整输入数据为 4D 张量
        N, V, C, H, W = x.shape
        x = x.view(N * V, C, H, W)  # 转换为 (N * V, C, H, W) 形状
        
        # backbone forward
        feat = self.backbone(x)
        # FPN forward
        out = self.neck(feat)
        return out

def export_backbone_neck(model, save_path, input_shape=(1, 6, 3, 256, 704)):
    """Export backbone + FPN (neck)"""
    model_with_backbone_neck = BackboneNeckModel(model)
    model_with_backbone_neck.eval()
    
    # 将模型移动到 GPU
    model_with_backbone_neck = model_with_backbone_neck.cuda()
    
    # 创建假输入数据，并将其移动到 GPU
    dummy_input = torch.randn(input_shape).cuda()
    
    # Apply normalization (将 mean 和 std 移到 GPU)
    mean = torch.tensor([123.675, 116.28, 103.53]).view(1, 3, 1, 1).cuda()
    std = torch.tensor([58.395, 57.12, 57.375]).view(1, 3, 1, 1).cuda()
    dummy_input = (dummy_input - mean) / std
    
    # 导出 ONNX 模型（不使用 dynamic_axes）
    torch.onnx.export(
        model_with_backbone_neck,
        dummy_input,
        save_path,
        input_names=['input'],
        output_names=['output_0', 'output_1', 'output_2', 'output_3'],
        opset_version=13,
        verbose=True,
        do_constant_folding=True
    )
    print(f"Backbone + Neck exported to {save_path}")

def main():
    # 配置文件路径和权重文件路径
    cfg_file = '/home/ubuntu/ycy/Sparse4D-3.0/projects/configs/sparse4dv3_temporal_r50_1x8_bs6_256x704.py'  # 这里是模型配置文件路径
    checkpoint_file = '/home/ubuntu/ycy/Sparse4D-3.0/work_dirs/sparse4dv3_temporal_r50_1x8_bs6_256x704/iter_9376.pth'  # 这里是模型权重路径
    output_onnx_path = '/home/ubuntu/ycy/Sparse4D-3.0/onnx_models/backbone_neck1.onnx'  # 导出文件路径
    
    # 加载配置文件
    cfg = Config.fromfile(cfg_file)
    
    # 构建模型
    model = build_model(cfg.model)
    
    # 加载模型权重
    checkpoint = torch.load(checkpoint_file, map_location='cpu')
    model.load_state_dict(checkpoint['state_dict'])
    
    # 将模型移动到 GPU
    model = model.cuda()
    
    # 导出 backbone + neck 部分
    export_backbone_neck(model, output_onnx_path)

if __name__ == '__main__':
    main()

onnx–>har

import hailo_sdk_client
from hailo_sdk_client import ClientRunner
import numpy as np
import os

# 1. Hailo 硬件架构
chosen_hw_arch = "hailo8"  # 使用 Hailo-8

# 2. 模型路径和文件名
onnx_model_name = ""  # 模型名称
onnx_path = ""  # ONNX 模型路径

# 输出文件路径
output_dir = ""
hailo_model_har_path = os.path.join(output_dir, f"{onnx_model_name}_hailo_model.har")
hailo_quantized_har_path = os.path.join(output_dir, f"{onnx_model_name}_hailo_quantized_model.har")
hailo_model_hef_path = os.path.join(output_dir, f"{onnx_model_name}.hef")

# 3. 确保输出目录存在
os.makedirs(output_dir, exist_ok=True)

# 4. 初始化 Hailo ClientRunner
try:
    runner = ClientRunner(hw_arch=chosen_hw_arch)
    print(f"✅ Initialized ClientRunner with hardware architecture: {chosen_hw_arch}")
except Exception as e:
    print(f"❌ Failed to initialize ClientRunner: {e}")
    exit(1)

# 5. 验证 ONNX 模型文件存在
if not os.path.exists(onnx_path):
    print(f"❌ ONNX model file not found: {onnx_path}")
    exit(1)
print(f"✅ ONNX model file found: {onnx_path}")

# 6. 指定输入节点的形状
net_input_shapes = {
    "input": (1, 6, 3, 256, 704)  # 输入节点名称是 "img"
}

# 7. 转换 ONNX 模型为 HAR
try:
    hn, npz = runner.translate_onnx_model(
        model=onnx_path,
        net_name=onnx_model_name,
        net_input_shapes=net_input_shapes
    )
    print(f"✅ Successfully translated ONNX model to HAR")
except Exception as e:
    print(f"❌ Failed to translate ONNX model to HAR: {e}")
    exit(1)

# 8. 保存 HAR 模型
try:
    runner.save_har(hailo_model_har_path)
    print(f"✅ HAR model saved to: {hailo_model_har_path}")
except Exception as e:
    print(f"❌ Failed to save HAR model: {e}")
    exit(1)

error
[warning] Cannot use graphviz, so no visualizations will be created
:white_check_mark: Initialized ClientRunner with hardware architecture: hailo8
:white_check_mark: ONNX model file found: /home/ubuntu/ycy/Sparse4D-3.0/onnx_models/backbone_neck1.onnx
[info] Translation started on ONNX model backbone_neck1
[info] Restored ONNX model backbone_neck1 (completion time: 00:00:00.26)
[info] Extracted ONNXRuntime meta-data for Hailo model (completion time: 00:00:01.76)
[info] Simplified ONNX model for a parsing retry attempt (completion time: 00:00:05.45)
:cross_mark: Failed to translate ONNX model to HAR: Invalid kernel shape for base conv layer base_conv2 (translated from /backbone/layer1/layer1.0/conv1/Conv).
Either the input shape doesn’t match the kernel shape, or the calculated groups number doesn’t match the expected ratio between kernel shape and input shape.
Kernel features: 384 Input features: 64 Groups: 0

2

Hey @user134 ,

That “invalid-kernel-shape” error happens because our parser expects standard 4-D convolutions, but your 5-D multi-view input is confusing the kernel-shape checker.

Solution: Convert to 4-D before export

Here’s what you need to do:

• Modify your PyTorch model: Remove the .view(N * V, …) operation and make your model accept pure 4-D input of shape (N * V, C, H, W) instead of 5-D

• Collapse dimensions during export: Flatten your batch and view dimensions into a single batch dimension before exporting to ONNX

• Use ONNX simplifier: Run the exported model through onnx-simplifier to clean up any reshape operations that might trip up our parser

• Standard translation: With the 4-D model, you can use our normal ONNX to HAR translation without any special flags

Let me know if you need any clarification!

3

Okay, thank you very much. It has been solved.