Hello!
I’ve been trying to run a model on a Hailo8L.
After some trial and error I could successfully optimize my model.
But now there is an issue during compilation:
[info] To achieve optimal performance, set the compiler_optimization_level to “max” by adding performance_param(compiler_optimization_level=max) to the model script. Note that this may increase compilation time.
[info] Loading network parameters
[info] Starting Hailo allocation and compilation flow
[error] Mapping Failed (allocation time: 0s)[error] Failed to produce compiled graph
No format for fc1 → conv50
I really have no idea what this could mean.
Please let me know if this issue seems familiar or point me in the right direction.
The tensor shapes seem okay to me.
The full trace:
---------------------------------------------------------------------------
BackendAllocatorException Traceback (most recent call last)
Cell In[4], line 1
----> 1 hef = runner.compile()
3 file_name = f"{model_name}.hef"
4 with open(file_name, "wb") as f:
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py:911, in ClientRunner.compile(self)
899 def compile(self):
900 """
901 DFC API for compiling current model to Hailo hardware.
902
(...)
909
910 """
--> 911 return self._compile()
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_common/states/states.py:16, in allowed_states.<locals>.wrap.<locals>.wrapped_func(self, *args, **kwargs)
12 if self._state not in states:
13 raise InvalidStateException(
14 f"The execution of {func.__name__} is not available under the state: {self._state.value}",
15 )
---> 16 return func(self, *args, **kwargs)
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py:1128, in ClientRunner._compile(self, fps, mapping_timeout, allocator_script_filename)
1122 self._logger.warning(
1123 f"Taking model script commands from {allocator_script_filename} and ignoring "
1124 f"previous allocation script commands",
1125 )
1126 self.load_model_script(allocator_script_filename)
-> 1128 serialized_hef = self._sdk_backend.compile(fps, self.model_script, mapping_timeout)
1130 self._auto_model_script = self._sdk_backend.get_auto_alls()
1131 self._state = States.COMPILED_SLIM_MODEL if orig_state in SLIM_STATES else States.COMPILED_MODEL
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1852, in SdkBackendCompilation.compile(self, fps, allocator_script, mapping_timeout)
1850 def compile(self, fps, allocator_script=None, mapping_timeout=None):
1851 self._model.fill_default_quantization_params(logger=self._logger)
-> 1852 hef, mapped_graph_file = self._compile(fps, allocator_script, mapping_timeout)
1853 # TODO: https://hailotech.atlassian.net/browse/SDK-31038
1854 if not SDKPaths().is_internal:
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1846, in SdkBackendCompilation._compile(self, fps, allocator_script, mapping_timeout)
1840 if not model_params and self.requires_quantized_weights:
1841 raise BackendRuntimeException(
1842 "Model requires quantized weights in order to run on HW, but none were given. "
1843 "Did you forget to quantize?",
1844 )
-> 1846 hef, mapped_graph_file, auto_alls = self.hef_full_build(fps, mapping_timeout, model_params, allocator_script)
1847 self._auto_alls = auto_alls
1848 return hef, mapped_graph_file
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1822, in SdkBackendCompilation.hef_full_build(self, fps, mapping_timeout, params, allocator_script)
1820 config_paths = ConfigPaths(self._hw_arch, self._model.name)
1821 config_paths.set_stage("inference")
-> 1822 auto_alls, self._hef_data, self._integrated_graph = allocator.create_mapping_and_full_build_hef(
1823 config_paths.get_path("network_graph"),
1824 config_paths.get_path("mapped_graph"),
1825 config_paths.get_path("compilation_output_proto"),
1826 params=params,
1827 allocator_script=allocator_script,
1828 compiler_statistics_path=config_paths.get_path("compiler_statistics"),
1829 nms_metadata=self._nms_metadata,
1830 har=self.har,
1831 alls_ignore_invalid_cmds=self._alls_ignore_invalid_cmds,
1832 )
1834 return self._hef_data, config_paths.get_path("mapped_graph"), auto_alls
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:764, in HailoToolsRunner.create_mapping_and_full_build_hef(self, network_graph_path, output_path, compilation_output_proto, agent, strategy, auto_mapping, params, expected_output_tensor, expected_pre_acts, network_inputs, network_outputs, allocator_script, allocator_script_mode, compiler_statistics_path, nms_metadata, har, alls_ignore_invalid_cmds)
759 if self.hn.net_params.clusters_placement != [[]]:
760 assert (
761 len(self.hn.net_params.clusters_placement) <= self._number_of_clusters
762 ), "Number of clusters in layer placements is larger than allowed number of clusters"
--> 764 self.call_builder(
765 network_graph_path,
766 output_path,
767 compilation_output_proto=compilation_output_proto,
768 agent=agent,
769 strategy=strategy,
770 exit_point=BuilderExitPoint.POST_CAT,
771 params=params,
772 expected_output_tensor=expected_output_tensor,
773 expected_pre_acts=expected_pre_acts,
774 network_inputs=network_inputs,
775 network_outputs=network_outputs,
776 allocator_script=allocator_script,
777 allocator_script_mode=allocator_script_mode,
778 compiler_statistics_path=compiler_statistics_path,
779 nms_metadata=nms_metadata,
780 har=har,
781 alls_ignore_invalid_cmds=alls_ignore_invalid_cmds,
782 )
784 return self._auto_alls, self._output_hef_data, self._output_integrated_pb_graph
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:696, in HailoToolsRunner.call_builder(self, network_graph_path, output_path, blind_deserialize, **kwargs)
694 sys.excepthook = _hailo_tools_exception_hook
695 try:
--> 696 self.run_builder(network_graph_path, output_path, **kwargs)
697 except BackendInternalException:
698 try:
File ~/hailo_env/lib/python3.10/site-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:570, in HailoToolsRunner.run_builder(self, network_graph_filename, output_filename, compilation_output_proto, agent, strategy, exit_point, params, expected_output_tensor, expected_pre_acts, network_inputs, network_outputs, allocator_script, allocator_script_mode, compiler_statistics_path, is_debug, nms_metadata, har, alls_ignore_invalid_cmds)
568 compiler_msg = e.hailo_tools_error
569 if compiler_msg:
--> 570 raise e.internal_exception("Compilation failed:", hailo_tools_error=compiler_msg) from None
571 else:
572 raise e.internal_exception("Compilation failed with unexpected crash") from None
BackendAllocatorException: Compilation failed: No format for fc1 -> conv50