I am trying to figure out how to implement inference of 1DCNN (Classification) on Raspberry Pi5 + Hailos 8 platform.
I have gone through the Model Zoo examples but it seems that these examples are related to video/ image inputs. However, in my case, I want to perform inference for 1D data type. Ideally, I would like to use the 1D-CNN model training using my computer and deploy it in the Pi5 platform. Furthermore, can Pytorch be used?
Hey @bijen.mali,
Welcome to the Hailo Community!
Yes, PyTorch can be used with Hailo. The process is as follows:
You need to convert your PyTorch file into a Hailo Executable Format (HEF) file using the Hailo Data Flow Compiler (DFC).
For more information on how to use the DFC, please check out the documentation on the Hailo Developer Zone website:
https://hailo.ai/developer-zone/documentation/
The DFC will be used on your PC to convert the PyTorch file into the HEF format.
After converting the file to HEF, you can then run it on the Raspberry Pi 5.
Best regards,
Thank you for the suggestion.
I have done the requirement
However, when I convert my ONNX file to HEF, I get the following error,
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:454, in Layer._check_valid_shape(self, shape)
449 raise UnsupportedModelError(
450 f"Unexpected dimension in shape {shape} at {self.full_name_msg}. "
451 f"Dimension must be of type ‘int’ or ‘long’“,
452 )
453 if any(dim == 0 for dim in shape):
→ 454 raise UnsupportedModelError(f"Unexpected zero dimension in shape {shape} at {self.full_name_msg}”)
UnsupportedModelError: Unexpected zero dimension in shape [-1, 2, 0, 64] at base conv layer base_conv1 (translated from /conv/Conv)
I have compared the performance of the Pytorch and ONNX file and both are performing as expected.
This is the structure of my Model is that helps.
class Model(nn.Module):
def __init__(self, input_channel, layers=[2, 2, 2, 2], num_classes=73):
super(Model, self).__init__()
self.inplanes3_1 = 64
self.conv = nn.Conv1d(input_channel, 64, kernel_size=3, stride=1, padding=1, bias=False)
self.bn = nn.BatchNorm1d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool1d(kernel_size=2, stride=2, padding=0)
self.layer3x3_11 = self._make_layer3(BasicBlock3x3, 64, layers[0], stride=2)
self.layer3x3_12 = self._make_layer3(BasicBlock3x3, 128, layers[1], stride=2)
self.layer3x3_21 = self._make_layer3(BasicBlock3x3, 64, layers[2], stride=2)
self.layer3x3_22 = self._make_layer3(BasicBlock3x3, 128, layers[3], stride=2)
self.avgpool = nn.AdaptiveAvgPool1d(1)
self.fc = nn.Linear(128 * BasicBlock3x3.expansion, num_classes)
def _make_layer3(self, block, planes, blocks, stride=2):
downsample = None
if stride != 1 or self.inplanes3_1 != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv1d(self.inplanes3_1, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
nn.BatchNorm1d(planes * block.expansion),
)
layers = []
layers.append(block(self.inplanes3_1, planes, stride, downsample))
self.inplanes3_1 = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes3_1, planes))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv(x)
x = self.bn(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer3x3_11(x)
x = self.layer3x3_12(x)
x = self.layer3x3_21(x)
x = self.layer3x3_22(x)
x = self.avgpool(x)
x = x.flatten(start_dim=1)
x = self.fc(x)
return x
Hi @bijen.mali,
I’ve modified your code a bit, and not it’s ok. Please see below.
class Model(nn.Module):
def __init__(self, input_channel, layers=[2, 2, 2, 2], num_classes=64):
super(Model, self).__init__()
self.inplanes3_1 = 64
self.conv = nn.Conv1d(input_channel, 64, kernel_size=3, stride=1, padding=1, bias=False)
self.bn = nn.BatchNorm1d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool1d(kernel_size=2, stride=2, padding=0)
self.avgpool = nn.AdaptiveAvgPool1d(1)
self.fc = nn.Linear(64, num_classes)
def forward(self, x):
x = self.conv(x)
x = self.bn(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.avgpool(x)
x = x.flatten(start_dim=1)
x = self.fc(x)
return x
I tried changed code but the problem persists.
This is the conversion code that I use.
from hailo_sdk_client import ClientRunner
chosen_hw_arch = 'hailo8'
onnx_model_name = 'TrainedModel6'
onnx_path = 'my_env/TrainedModel6.onnx'
runner = ClientRunner(hw_arch=chosen_hw_arch)
hn, npz = runner.translate_onnx_model(
onnx_path,
onnx_model_name,
start_node_names=['input'],
end_node_names=['output'],
net_input_shapes={'input': [1, 1, 200]}
)
However, I still get this error
[info] Translation started on ONNX model TrainedModel6
[info] Restored ONNX model TrainedModel6 (completion time: 00:00:00.01)
[info] Extracted ONNXRuntime meta-data for Hailo model (completion time: 00:00:00.01)
[info] Simplified ONNX model for a parsing retry attempt (completion time: 00:00:00.02)
---------------------------------------------------------------------------
UnsupportedModelError Traceback (most recent call last)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:220, in Parser.translate_onnx_model(self, model, net_name, start_node_names, end_node_names, net_input_shapes, augmented_path, disable_shape_inference, disable_rt_metadata_extraction, net_input_format, **kwargs)
219 try:
--> 220 parsing_results = self._parse_onnx_model_to_hn(
221 onnx_model=onnx_model,
222 net_name=valid_net_name,
223 start_node_names=start_node_names,
224 end_node_names=end_node_names,
225 net_input_shapes=net_input_shapes,
226 disable_shape_inference=disable_shape_inference,
227 net_input_format=net_input_format,
228 )
230 except Exception as e:
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:300, in Parser._parse_onnx_model_to_hn(self, onnx_model, net_name, start_node_names, end_node_names, net_input_shapes, disable_shape_inference, net_input_format, **kwargs)
298 self._logger.warning(f"ONNX shape inference failed: {e!s}")
--> 300 return self.parse_model_to_hn(
301 onnx_model,
302 None,
303 net_name,
304 start_node_names,
305 end_node_names,
306 nn_framework=NNFramework.ONNX,
307 output_shapes=output_shapes,
308 net_input_format=net_input_format,
309 **kwargs,
310 )
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:351, in Parser.parse_model_to_hn(self, model, values, net_name, start_node_names, end_node_names, nn_framework, output_shapes, net_input_format, rename_layers_by_blocks)
349 raise BackendRuntimeException(f"Unsupported NN framework {nn_framework}")
--> 351 fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
352 hailo_nn = fuser.convert_model()
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/model_translator/translator.py:88, in HailoNNConverter.convert_model(self)
87 self._handle_tokens_matmul()
---> 88 self._calculate_shapes(validate_shapes=False)
89 self._add_output_layers()
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py:194, in ONNXConverter._calculate_shapes(self, validate_shapes)
193 self._update_meta_graph()
--> 194 self._layers_graph.calculate_shapes(meta_edges_graph=self._meta_graph, validate_shapes=validate_shapes)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hailo_nn.py:755, in HailoNN.calculate_shapes(self, meta_edges_graph, validate_shapes)
753 layer.output_copies = self.get_number_of_successors(layer)
--> 755 layer.update_output_shapes(hn_stage=self.net_params.stage, validate_shapes=validate_shapes)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:458, in Layer.update_output_shapes(self, **kwargs)
457 output_shape = self._calc_output_shape()
--> 458 self.output_shapes = [output_shape[:] for _ in range(self.output_copies)]
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:487, in Layer.output_shapes(self, output_shapes)
486 for shape in output_shapes:
--> 487 self._append_output_shape(shape)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:461, in Layer._append_output_shape(self, output_shape)
460 def _append_output_shape(self, output_shape):
--> 461 self._check_valid_shape(output_shape)
462 self.append_output_shape(output_shape)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:454, in Layer._check_valid_shape(self, shape)
453 if any(dim == 0 for dim in shape):
--> 454 raise UnsupportedModelError(f"Unexpected zero dimension in shape {shape} at {self.full_name_msg}")
UnsupportedModelError: Unexpected zero dimension in shape [-1, 4, 0, 64] at base conv layer base_conv1 (translated from /conv/Conv)
During handling of the above exception, another exception occurred:
UnsupportedModelError Traceback (most recent call last)
Cell In[4], line 2
1 runner = ClientRunner(hw_arch=chosen_hw_arch)
----> 2 hn, npz = runner.translate_onnx_model(
3 onnx_path,
4 onnx_model_name,
5 start_node_names=['input'],
6 end_node_names=['output'],
7 net_input_shapes={'input': [1, 1, 200]}
8 )
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/states/states.py:16, in allowed_states.<locals>.wrap.<locals>.wrapped_func(self, *args, **kwargs)
12 if self._state not in states:
13 raise InvalidStateException(
14 f"The execution of {func.__name__} is not available under the state: {self._state.value}",
15 )
---> 16 return func(self, *args, **kwargs)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py:1158, in ClientRunner.translate_onnx_model(self, model, net_name, start_node_names, end_node_names, net_input_shapes, augmented_path, disable_shape_inference, disable_rt_metadata_extraction, net_input_format, **kwargs)
1115 """
1116 DFC API for parsing an ONNX model. This creates a runner with loaded HN (model) and
1117 parameters.
(...)
1155
1156 """
1157 parser = Parser()
-> 1158 parser.translate_onnx_model(
1159 model=model,
1160 net_name=net_name,
1161 start_node_names=start_node_names,
1162 end_node_names=end_node_names,
1163 net_input_shapes=net_input_shapes,
1164 augmented_path=augmented_path,
1165 disable_shape_inference=disable_shape_inference,
1166 disable_rt_metadata_extraction=disable_rt_metadata_extraction,
1167 net_input_format=net_input_format,
1168 **kwargs,
1169 )
1171 return self._finalize_parsing(parser.return_data)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:260, in Parser.translate_onnx_model(self, model, net_name, start_node_names, end_node_names, net_input_shapes, augmented_path, disable_shape_inference, disable_rt_metadata_extraction, net_input_format, **kwargs)
257 milestone = self._format_time_milestone(start_time)
258 self._logger.info(f"Simplified ONNX model for a parsing retry attempt (completion time: {milestone})")
--> 260 parsing_results = self._parse_onnx_model_to_hn(
261 onnx_model=simplified_model,
262 net_name=valid_net_name,
263 start_node_names=start_node_names,
264 end_node_names=end_node_names,
265 net_input_shapes=net_input_shapes,
266 disable_shape_inference=disable_shape_inference,
267 net_input_format=net_input_format,
268 **kwargs,
269 )
271 milestone = self._format_time_milestone(start_time)
272 self._logger.info(f"Translation completed on ONNX model {valid_net_name} (completion time: {milestone})")
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:300, in Parser._parse_onnx_model_to_hn(self, onnx_model, net_name, start_node_names, end_node_names, net_input_shapes, disable_shape_inference, net_input_format, **kwargs)
297 except Exception as e:
298 self._logger.warning(f"ONNX shape inference failed: {e!s}")
--> 300 return self.parse_model_to_hn(
301 onnx_model,
302 None,
303 net_name,
304 start_node_names,
305 end_node_names,
306 nn_framework=NNFramework.ONNX,
307 output_shapes=output_shapes,
308 net_input_format=net_input_format,
309 **kwargs,
310 )
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/parser/parser.py:351, in Parser.parse_model_to_hn(self, model, values, net_name, start_node_names, end_node_names, nn_framework, output_shapes, net_input_format, rename_layers_by_blocks)
348 else:
349 raise BackendRuntimeException(f"Unsupported NN framework {nn_framework}")
--> 351 fuser = HailoNNFuser(converter.convert_model(), net_name, converter.end_node_names)
352 hailo_nn = fuser.convert_model()
353 hailo_nn.validate_stage(HnStage.HN)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/model_translator/translator.py:88, in HailoNNConverter.convert_model(self)
86 self._handle_inner_product_matmul()
87 self._handle_tokens_matmul()
---> 88 self._calculate_shapes(validate_shapes=False)
89 self._add_output_layers()
90 self._handle_fused_layers()
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_client/model_translator/onnx_translator/onnx_translator.py:194, in ONNXConverter._calculate_shapes(self, validate_shapes)
192 def _calculate_shapes(self, validate_shapes=True):
193 self._update_meta_graph()
--> 194 self._layers_graph.calculate_shapes(meta_edges_graph=self._meta_graph, validate_shapes=validate_shapes)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hailo_nn.py:755, in HailoNN.calculate_shapes(self, meta_edges_graph, validate_shapes)
752 else:
753 layer.output_copies = self.get_number_of_successors(layer)
--> 755 layer.update_output_shapes(hn_stage=self.net_params.stage, validate_shapes=validate_shapes)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:458, in Layer.update_output_shapes(self, **kwargs)
456 def update_output_shapes(self, **kwargs):
457 output_shape = self._calc_output_shape()
--> 458 self.output_shapes = [output_shape[:] for _ in range(self.output_copies)]
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:487, in Layer.output_shapes(self, output_shapes)
485 self._output_shapes = []
486 for shape in output_shapes:
--> 487 self._append_output_shape(shape)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:461, in Layer._append_output_shape(self, output_shape)
460 def _append_output_shape(self, output_shape):
--> 461 self._check_valid_shape(output_shape)
462 self.append_output_shape(output_shape)
File ~/my_env/lib/python3.10/site-packages/hailo_sdk_common/hailo_nn/hn_layers/layer.py:454, in Layer._check_valid_shape(self, shape)
449 raise UnsupportedModelError(
450 f"Unexpected dimension in shape {shape} at {self.full_name_msg}. "
451 f"Dimension must be of type 'int' or 'long'",
452 )
453 if any(dim == 0 for dim in shape):
--> 454 raise UnsupportedModelError(f"Unexpected zero dimension in shape {shape} at {self.full_name_msg}")
UnsupportedModelError: Unexpected zero dimension in shape [-1, 4, 0, 64] at base conv layer base_conv1 (translated from /conv/Conv)
I have also attached the onnx structure I viewed using netron.
I think that you get a dimmension 0 in one of the middle tensors. Try to do onnxsim on your ONNX and see the inter layer tensors.
I have done the onnxsim.
I looked the node information using the following code.
import onnx
import onnxruntime as ort
import numpy as np
model_path = 'SimplifiedModel.onnx'
model = onnx.load(model_path)
session = ort.InferenceSession(model_path)
input_shape = [1, 1, 200]
input_data = np.random.random(input_shape).astype(np.float32)
input_name = session.get_inputs()[0].name
outputs = session.run(None, {input_name: input_data})
def extract_tensor_shapes_from_initializer(model):
initializer_shapes = {}
for initializer in model.graph.initializer:
shape = [dim for dim in initializer.dims]
initializer_shapes[initializer.name] = shape
return initializer_shapes
def get_tensor_shapes(model):
tensor_shapes = {}
initializer_shapes = extract_tensor_shapes_from_initializer(model)
for tensor in model.graph.value_info:
shape = [dim.dim_value for dim in tensor.type.tensor_type.shape.dim]
tensor_shapes[tensor.name] = shape
for output in model.graph.output:
shape = [dim.dim_value for dim in output.type.tensor_type.shape.dim]
tensor_shapes[output.name] = shape
tensor_shapes.update(initializer_shapes)
return tensor_shapes
tensor_shapes = get_tensor_shapes(model)
for node in model.graph.node:
print(f"Node: {node.name}, OpType: {node.op_type}")
for input in node.input:
if input in tensor_shapes:
print(f" Input: {input}, Shape: {tensor_shapes[input]}")
else:
print(f" Input: {input}, Shape: Unknown")
for output in node.output:
if output in tensor_shapes:
print(f" Output: {output}, Shape: {tensor_shapes[output]}")
else:
print(f" Output: {output}, Shape: Unknown")
And I get the results as,
Input: input, Shape: Unknown
Input: onnx::Conv_17, Shape: [64, 1, 5]
Input: onnx::Conv_18, Shape: [64]
Output: /conv/Conv_output_0, Shape: [1, 64, 101]
Node: /relu/Relu, OpType: Relu
Input: /conv/Conv_output_0, Shape: [1, 64, 101]
Output: /relu/Relu_output_0, Shape: [1, 64, 101]
Node: /maxpool/MaxPool, OpType: MaxPool
Input: /relu/Relu_output_0, Shape: [1, 64, 101]
Output: /maxpool/MaxPool_output_0, Shape: [1, 64, 50]
Node: /avgpool/GlobalAveragePool, OpType: GlobalAveragePool
Input: /maxpool/MaxPool_output_0, Shape: [1, 64, 50]
Output: /avgpool/GlobalAveragePool_output_0, Shape: [1, 64, 1]
Node: /Flatten, OpType: Flatten
Input: /avgpool/GlobalAveragePool_output_0, Shape: [1, 64, 1]
Output: /Flatten_output_0, Shape: [1, 64]
Node: /fc/Gemm, OpType: Gemm
Input: /Flatten_output_0, Shape: [1, 64]
Input: fc.weight, Shape: [73, 64]
Input: fc.bias, Shape: [73]
Output: output, Shape: [1, 73]
Netron of the simplified ONNX is the same as the one I provided earlier.
Do you have any recommendation?
something in your shapes looks strange, please try to keep 4d vectors along the net
You were correct, I had to use the 4D vectors. For this I converter my 1D data to 4D.
The updated Model looks like this.
class Model(nn.Module):
def __init__(self, input_channel, num_classes=73):
super(Model, self).__init__()
self.conv1 = nn.Conv2d(input_channel, 64, kernel_size=(1, 5), stride=(1, 2), padding=(0, 3), bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=(1, 3), stride=(1, 2), padding=(0, 0))
self.conv2 = nn.Conv2d(64, 128, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)
self.bn2 = nn.BatchNorm2d(128)
self.conv3 = nn.Conv2d(128, 256, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)
self.bn3 = nn.BatchNorm2d(256)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(256, num_classes)
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.conv2(x)
x = self.bn2(x)
x = self.relu(x)
x = self.conv3(x)
x = self.bn3(x)
x = self.relu(x)
x = self.avgpool(x)
x = x.flatten(start_dim=1)
x = self.fc(x)
return x
After the restructuring, I did not even need to use onnxsim for simplification.
Next for the conversion to HAR file, optimization and the compile to HEF, I ended up using this code from
import numpy as np
import os
from hailo_sdk_client import ClientRunner
#Define model information
model_name = 'Model'
onnx_path = 'my_env/Model.onnx'
start_node = 'input'
end_node = 'output'
input_shape = {'input': [1, 1, 1, 200]}
chosen_hw_arch = 'hailo8'
input_height = 1
input_width = 200
input_ch = 1
alls_lines = [
'model_optimization_flavor(optimization_level=0, compression_level=1)\n',
'resources_param(max_control_utilization=1.0, max_compute_utilization=1.0,max_memory_utilization=1.0)\n',
'performance_param(fps=250)\n'
]
#Parsing
runner = ClientRunner(hw_arch=chosen_hw_arch)
hn, npz = runner.translate_onnx_model(onnx_path, model_name, start_node_names=[start_node], end_node_names=end_node, net_input_shapes=input_shape)
parsed_model_har_path = f'{model_name}_parsed_model.har'
runner.save_har(parsed_model_har_path)
#Optimize
calibData = np.random.randint(0, 255, (1024, input_height, input_width, input_ch)).transpose(0, 3, 2, 1)
runner.load_model_script(''.join(alls_lines))
runner.optimize(calibData)
quantized_model_har_path = f'{model_name}_quantized_model.har'
runner.save_har(quantized_model_har_path)
#Compile
hef = runner.compile()
file_name = f'{model_name}.hef'
with open(file_name, 'wb') as f:
f.write(hef)
compiled_model_har_path = f'{model_name}_compiled_model.har'
runner.save_har(compiled_model_har_path)
I am not user if the output is correct because I am using 1D data and not image. So for optimization and quantization I did not find ant relevant documentations. However, conversion to HAR seems to be completed.
