How to parsing the LSTM.ONNX to HAR files

General
1

Dear all members,

I am currently trying to convert an LSTM-based .pt model into an HEF file for deployment on the Hailo-8 accelerator.

During this process, I successfully converted the .pt model to an ONNX format and verified its functionality using onnxruntime.InferenceSession.

However, I encountered errors while converting the ONNX file to a HAR format.

If anyone has experience with LSTM model conversion, please share your insights regarding the following code and error messages.

<>

class LSTM(nn.Module):
def init(self, num_sensors, hidden_units, num_layers):
super().init()
self.num_sensors = num_sensors # this is the number of features
self.hidden_units = hidden_units
self.num_layers = num_layers

    self.lstm = nn.LSTM(
        input_size=num_sensors,
        hidden_size=hidden_units,
        batch_first=True,
        num_layers=self.num_layers
    )

    self.linear = nn.Linear(in_features=self.hidden_units, out_features=1)

def forward(self, x, h0, c0):
    # batch_size = x.shape[0]
    # h0 = torch.zeros(self.num_layers, batch_size, self.hidden_units).requires_grad_().to(device)
    # c0 = torch.zeros(self.num_layers, batch_size, self.hidden_units).requires_grad_().to(device)
    
    _, (hn, cn) = self.lstm(x, (h0, c0))
    
    # out = self.linear(hn[0]).flatten()  # First dim of Hn is num_layers, which is set to 1 above.
    out = self.linear(hn[-1]).flatten()  # First dim of Hn is num_layers, which is set to 1 above.
    
    return out, hn, cn

<>

더미 입력 생성 (배치 크기 1)

batch_size = 1
sequence_length = 30
input_size = 14

N_HIDDEN = 64 # NUMBER OF HIDDEN STATES
N_LAYER = 1 # NUMBER OF LSTM LAYERS

dummy_input = torch.randn(batch_size, sequence_length, input_size).to(device)
h0 = torch.zeros(N_LAYER, batch_size, N_HIDDEN).to(device)
c0 = torch.zeros(N_LAYER, batch_size, N_HIDDEN).to(device)

ONNX로 변환 및 저장

onnx_file_path = “./LSTM_sample_model_v2.onnx”

print(testX_tensor[:1].to(device).to(torch.float32).shape)
torch.onnx.export(
model, # 학습된 모델
(dummy_input, h0, c0),
# testX_tensor[:1].to(device).to(torch.float32), # 입력 예제
# testX_tensor.to(device).to(torch.float32), # 입력 예제
onnx_file_path, # 저장 경로
export_params=True, # 학습된 파라미터 포함
input_names=[‘input’, ‘h0’, ‘c0’], # h0, c0 추가
output_names=[‘output’, ‘hn’, ‘cn’], # LSTM의 새로운 상태도 출력
# dynamic_axes={‘input’: {0: ‘batch_size’}, ‘output’: {0: ‘batch_size’}},
# dynamic_axes=None,
dynamic_axes={
‘input’: {0: ‘batch_size’},
‘h0’: {1: ‘batch_size’},
‘c0’: {1: ‘batch_size’},
‘output’: {0: ‘batch_size’},
‘hn’: {1: ‘batch_size’},
‘cn’: {1: ‘batch_size’}
},
do_constant_folding=True, # 상수 폴딩 활성화 (최적화)
keep_initializers_as_inputs=False # 초기화를 입력에서 제거
)

<< have problem code to HAR>
from hailo_sdk_client import ClientRunner

chosen_hw_arch = “hailo8”

For Hailo-15 devices, use ‘hailo15h’

For Mini PCIe modules or Hailo-8R devices, use ‘hailo8r’

onnx_model_name = “LSTM_sample_model_v2”
onnx_path = “./LSTM_sample_model_v2.onnx”

runner = ClientRunner(hw_arch=chosen_hw_arch)

hn, npz = runner.translate_onnx_model(
onnx_path,
start_node_names=[“input”],
# start_node_names=[‘input’, ‘h0’, ‘c0’],
end_node_names=[“output”],
# end_node_names=[‘output’, ‘hn’, ‘cn’],
net_input_shapes={“input”: [batch_size, 30, 14]},
# net_input_shapes={“input”: [batch_size, 30, 14], “h0”: [1, batch_size, 64], “c0”: [1, batch_size, 64]},
)

<<Faced Error 1 >>

[warning] ONNX shape inference failed: [ONNXRuntimeError] : 2 : INVALID_ARGUMENT : Invalid model. Node input ‘h0’ is not a graph input, initializer, or output of a previous node.
[info] Unable to simplify the model: list index out of range
Traceback (most recent call last):
File “/home/jskang/Codings/LSTM_test/Conv_HEF_v2.py”, line 72, in
hn, npz = runner.translate_onnx_model(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_common/states/states.py”, line 16, in wrapped_func
return func(self, *args, **kwargs)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/runner/client_runner.py”, line 1158, in translate_onnx_model
parser.translate_onnx_model(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 258, in translate_onnx_model
raise e from None
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 235, in translate_onnx_model
parsing_results = self._parse_onnx_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 316, in _parse_onnx_model_to_hn
return self.parse_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 367, in parse_model_to_hn
fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/translator.py”, line 83, in convert_model
self._create_layers()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py”, line 39, in _create_layers
self._update_vertices_info()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py”, line 217, in _update_vertices_info
node.update_output_format()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 478, in update_output_format
self.update_reshape_output_format(input_format)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 415, in update_reshape_output_format
elif self.is_groups_to_spatial_flatten():
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 3191, in is_groups_to_spatial_flatten
input_shape = self.get_input_shapes(convert_to_nhwc=False)[0]
IndexError: list index out of range

<<Faced Error 2>>
[info] Simplified ONNX model for a parsing retry attempt (completion time: 00:00:00.03)
Traceback (most recent call last):
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 235, in translate_onnx_model
parsing_results = self._parse_onnx_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 316, in _parse_onnx_model_to_hn
return self.parse_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 367, in parse_model_to_hn
fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/translator.py”, line 83, in convert_model
self._create_layers()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py”, line 40, in _create_layers
self._add_direct_layers()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py”, line 122, in _add_direct_layers
self._layer_callback_from_vertex(vertex)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py”, line 460, in _layer_callback_from_vertex
consumed_vertices = self._create_lstm_layer(vertex)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py”, line 2041, in _create_lstm_layer
forward_params, backward_params, direction, consumed_vertices = vertex.get_lstm_info()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 1330, in get_lstm_info
info_dict[param] = const.parse_raw_data()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 620, in parse_raw_data
parsed_data = numpy_helper.to_array(self._info.attribute[0].t)
AttributeError: attribute

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File “/home/jskang/Codings/LSTM_test/Conv_HEF_v2.py”, line 72, in
hn, npz = runner.translate_onnx_model(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_common/states/states.py”, line 16, in wrapped_func
return func(self, *args, **kwargs)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/runner/client_runner.py”, line 1158, in translate_onnx_model
parser.translate_onnx_model(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 276, in translate_onnx_model
parsing_results = self._parse_onnx_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 316, in _parse_onnx_model_to_hn
return self.parse_model_to_hn(
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py”, line 367, in parse_model_to_hn
fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/translator.py”, line 83, in convert_model
self._create_layers()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py”, line 40, in _create_layers
self._add_direct_layers()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py”, line 122, in _add_direct_layers
self._layer_callback_from_vertex(vertex)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py”, line 460, in _layer_callback_from_vertex
consumed_vertices = self._create_lstm_layer(vertex)
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py”, line 2041, in _create_lstm_layer
forward_params, backward_params, direction, consumed_vertices = vertex.get_lstm_info()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 1330, in get_lstm_info
info_dict[param] = const.parse_raw_data()
File “/home/jskang/venv_JSK/GRPC_JSK_v2_RT/lib/python3.9/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py”, line 620, in parse_raw_data
parsed_data = numpy_helper.to_array(self._info.attribute[0].t)
AttributeError: attribute

1 Like
2

I have the same problem you mention in error 2. Any help for this?
has anyone parsed LSTM.onnx model to HAR?