Need help to convert a LSTM model

General
,
1

I am trying to convert a LSTM model to HEF format, but I get this error

[info] Translation started on ONNX model model
[info] Restored ONNX model model (completion time: 00:00:00.02)
[info] Extracted ONNXRuntime meta-data for Hailo model (completion time: 00:00:00.08)
[info] Simplified ONNX model for a parsing retry attempt (completion time: 00:00:00.15)
Traceback (most recent call last):
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 235, in translate_onnx_model
    parsing_results = self._parse_onnx_model_to_hn(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 316, in _parse_onnx_model_to_hn
    return self.parse_model_to_hn(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 367, in parse_model_to_hn
    fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/translator.py", line 83, in convert_model
    self._create_layers()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py", line 40, in _create_layers
    self._add_direct_layers()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py", line 122, in _add_direct_layers
    self._layer_callback_from_vertex(vertex)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py", line 460, in _layer_callback_from_vertex
    consumed_vertices = self._create_lstm_layer(vertex)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py", line 2041, in _create_lstm_layer
    forward_params, backward_params, direction, consumed_vertices = vertex.get_lstm_info()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py", line 1330, in get_lstm_info
    info_dict[param] = const.parse_raw_data()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py", line 620, in parse_raw_data
    parsed_data = numpy_helper.to_array(self._info.attribute[0].t)
IndexError: list index (0) out of range

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/azureuser/projects/misc/HEF_Convertor/hef_convertor.py", line 9, in <module>
    hn, npz = runner.translate_onnx_model(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_common/states/states.py", line 16, in wrapped_func
    return func(self, *args, **kwargs)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py", line 1158, in translate_onnx_model
    parser.translate_onnx_model(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 276, in translate_onnx_model
    parsing_results = self._parse_onnx_model_to_hn(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 316, in _parse_onnx_model_to_hn
    return self.parse_model_to_hn(
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py", line 367, in parse_model_to_hn
    fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/translator.py", line 83, in convert_model
    self._create_layers()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py", line 40, in _create_layers
    self._add_direct_layers()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/edge_nn_translator.py", line 122, in _add_direct_layers
    self._layer_callback_from_vertex(vertex)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py", line 460, in _layer_callback_from_vertex
    consumed_vertices = self._create_lstm_layer(vertex)
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py", line 2041, in _create_lstm_layer
    forward_params, backward_params, direction, consumed_vertices = vertex.get_lstm_info()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py", line 1330, in get_lstm_info
    info_dict[param] = const.parse_raw_data()
  File "/home/azureuser/miniconda3/envs/Hailo_convert/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_graph.py", line 620, in parse_raw_data
    parsed_data = numpy_helper.to_array(self._info.attribute[0].t)
IndexError: list index (0) out of range

This is the code I used to convert

from hailo_sdk_client import ClientRunner

model_name = 'LSTM'

chosen_hw_arch = "hailo8l"
onnx_path = f"{model_name}.onnx"

runner = ClientRunner(hw_arch=chosen_hw_arch)
hn, npz = runner.translate_onnx_model(
		onnx_path,
)
hailo_model_har_name = f"{model_name}.har"
runner.save_har(hailo_model_har_name)

#Compiling HAR to HER(Hailo Executable File)
hailo_model_har_name = f"{model_name}.har"
runner = ClientRunner(har=hailo_model_har_name)
hef = runner.compile()

file_name = f"{model_name}.hef"

with open(file_name, "wb") as f:
    f.write(hef)

Here is the model I built

import torch
import torch.nn as nn

class LSTM(nn.Module):
    def __init__(self, input_dim=51, hidden_dim=128, num_layers=2, dropout=0.2):
        super(FallDetectionLSTM, self).__init__()
        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        
        self.input_bn = nn.BatchNorm1d(input_dim)
        
        self.lstm = nn.LSTM(
            input_size=input_dim,
            hidden_size=hidden_dim,
            num_layers=num_layers,
            batch_first=True,
            bidirectional=True,
            dropout=dropout if num_layers > 1 else 0
        )
        
        self.attention = nn.Sequential(
            nn.Linear(hidden_dim * 2, hidden_dim),
            nn.Tanh(),
            nn.Linear(hidden_dim, 1),
            nn.Softmax(dim=1)
        )
        
        self.fc_layers = nn.Sequential(
            nn.Linear(hidden_dim * 2, hidden_dim),
            nn.BatchNorm1d(hidden_dim),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, hidden_dim // 2),
            nn.BatchNorm1d(hidden_dim // 2),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim // 2, 1)
        )

    def attention_net(self, lstm_output):
        attention_weights = self.attention(lstm_output)
        context_vector = attention_weights * lstm_output
        context_vector = torch.sum(context_vector, dim=1)
        return context_vector

    def forward(self, x):
        batch_size, seq_length, _ = x.size()
        x = x.view(-1, self.input_dim)
        x = self.input_bn(x)
        x = x.view(batch_size, seq_length, -1)
        
        lstm_out, (hidden, cell) = self.lstm(x)
        
        attention_output = self.attention_net(lstm_out)
        
        output = self.fc_layers(attention_output)
        
        return torch.sigmoid(output)
3

Guys any help for this?

4

I’m seeing the same problem when targeting an LSTM model to the Hailo15h during the parsing phase.

I suspect it may be due to the loop unrolling that has to occur to support LSTMs on the Hailo -

Since Hailo does not allow feedback loops, those layers are supported by the Unrolling technique, which duplicates each RNN or LSTM block by sequence length times.

Maybe someone from Hailo can confirm this?