Error compiling model with custom dataset

I’m following the tutorial using the “hailo tutorial” command which opens a Jupyter notebook.
I generated the .har model, optimized it with full_precision, and then used a custom dataset.
The .onnx model I used was also trained with a custom dataset containing different images.
I’m encountering an error when trying to compile the model as shown below:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[7], line 1
----> 1 hef = runner.compile()
      3 file_name = f"{model_name}.hef"
      4 with open(file_name, "wb") as f:

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/runner/client_runner.py:886, in ClientRunner.compile(self)
    874 def compile(self):
    875     """
    876     DFC API for compiling current model to Hailo hardware.
    877 
   (...)
    884 
    885     """
--> 886     return self._compile()

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_common/states/states.py:16, in allowed_states.<locals>.wrap.<locals>.wrapped_func(self, *args, **kwargs)
     12 if self._state not in states:
     13     raise InvalidStateException(
     14         f"The execution of {func.__name__} is not available under the state: {self._state.value}",
     15     )
---> 16 return func(self, *args, **kwargs)

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/runner/client_runner.py:1094, in ClientRunner._compile(self, fps, mapping_timeout, allocator_script_filename)
   1088         self._logger.warning(
   1089             f"Taking model script commands from {allocator_script_filename} and ignoring "
   1090             f"previous allocation script commands",
   1091         )
   1092     self.load_model_script(allocator_script_filename)
-> 1094 serialized_hef = self._sdk_backend.compile(fps, self.model_script, mapping_timeout)
   1096 self._auto_model_script = self._sdk_backend.get_auto_alls()
   1097 self._state = States.COMPILED_SLIM_MODEL if orig_state in SLIM_STATES else States.COMPILED_MODEL

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1655, in SdkBackendCompilation.compile(self, fps, allocator_script, mapping_timeout)
   1653 def compile(self, fps, allocator_script=None, mapping_timeout=None):
   1654     self._model.fill_default_quantization_params(logger=self._logger)
-> 1655     hef, mapped_graph_file = self._compile(fps, allocator_script, mapping_timeout)
   1656     # TODO: https://hailotech.atlassian.net/browse/SDK-31038
   1657     if not SDKPaths().is_internal:

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1649, in SdkBackendCompilation._compile(self, fps, allocator_script, mapping_timeout)
   1643 if not model_params and self.requires_quantized_weights:
   1644     raise BackendRuntimeException(
   1645         "Model requires quantized weights in order to run on HW, but none were given. "
   1646         "Did you forget to quantize?",
   1647     )
-> 1649 hef, mapped_graph_file, auto_alls = self.hef_full_build(fps, mapping_timeout, model_params, allocator_script)
   1650 self._auto_alls = auto_alls
   1651 return hef, mapped_graph_file

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1625, in SdkBackendCompilation.hef_full_build(self, fps, mapping_timeout, params, allocator_script)
   1623 config_paths = ConfigPaths(self._hw_arch, self._model.name)
   1624 config_paths.set_stage("inference")
-> 1625 auto_alls, self._hef_data, self._integrated_graph = allocator.create_mapping_and_full_build_hef(
   1626     config_paths.get_path("network_graph"),
   1627     config_paths.get_path("mapped_graph"),
   1628     config_paths.get_path("compilation_output_proto"),
   1629     params=params,
   1630     allocator_script=allocator_script,
   1631     compiler_statistics_path=config_paths.get_path("compiler_statistics"),
   1632     nms_metadata=self._nms_metadata,
   1633     har=self.har,
   1634     alls_ignore_invalid_cmds=self._alls_ignore_invalid_cmds,
   1635 )
   1637 return self._hef_data, config_paths.get_path("mapped_graph"), auto_alls

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:772, in HailoToolsRunner.create_mapping_and_full_build_hef(self, network_graph_path, output_path, compilation_output_proto, agent, strategy, auto_mapping, params, expected_output_tensor, expected_pre_acts, allocator_script, allocator_script_mode, compiler_statistics_path, nms_metadata, har, alls_ignore_invalid_cmds)
    767 if self.hn.net_params.clusters_placement != [[]]:
    768     assert (
    769         len(self.hn.net_params.clusters_placement) <= self._number_of_clusters
    770     ), "Number of clusters in layer placements is larger than allowed number of clusters"
--> 772 self.call_builder(
    773     network_graph_path,
    774     output_path,
    775     compilation_output_proto=compilation_output_proto,
    776     agent=agent,
    777     strategy=strategy,
    778     exit_point=BuilderExitPoint.POST_CAT,
    779     params=params,
    780     expected_output_tensor=expected_output_tensor,
    781     expected_pre_acts=expected_pre_acts,
    782     allocator_script=allocator_script,
    783     allocator_script_mode=allocator_script_mode,
    784     compiler_statistics_path=compiler_statistics_path,
    785     nms_metadata=nms_metadata,
    786     har=har,
    787     alls_ignore_invalid_cmds=alls_ignore_invalid_cmds,
    788 )
    790 return self._auto_alls, self._output_hef_data, self._output_integrated_pb_graph

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:706, in HailoToolsRunner.call_builder(self, network_graph_path, output_path, blind_deserialize, **kwargs)
    704 sys.excepthook = _hailo_tools_exception_hook
    705 try:
--> 706     self.run_builder(network_graph_path, output_path, **kwargs)
    707 except BackendInternalException:
    708     try:

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:559, in HailoToolsRunner.run_builder(self, network_graph_filename, output_filename, compilation_output_proto, agent, strategy, exit_point, params, expected_output_tensor, expected_pre_acts, allocator_script, allocator_script_mode, compiler_statistics_path, is_debug, nms_metadata, har, alls_ignore_invalid_cmds)
    556 self._extract_tensors(paths, params, expected_output_tensor, expected_pre_acts)
    558 try:
--> 559     self._run_hailo_tools(paths)
    560 except HailoToolsException as e:
    561     print("\033[?25h")  # Bring back the cursur if it's still hidden

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:314, in HailoToolsRunner._run_hailo_tools(self, tool_args)
    313 def _run_hailo_tools(self, tool_args):
--> 314     self._output_hef_data, self._output_integrated_pb_graph = run_hailo_tools(
    315         tool_args,
    316         "compiler",
    317         client=self._client,
    318         server=self._server,
    319         builder_pb_input=self._input_pb_string,
    320     )

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:215, in run_hailo_tools(tool_args, exe_name, client, server, builder_pb_input)
    213 def run_hailo_tools(tool_args, exe_name, client=None, server=None, builder_pb_input=None):
    214     hailo_tools_path = SDKPaths().join_hailo_tools_path("build/" + exe_name)
--> 215     return run_tool_from_binary(
    216         hailo_tools_path,
    217         tool_args,
    218         client=client,
    219         server=server,
    220         builder_pb_input=builder_pb_input,
    221     )

File /usr/local/lib/python3.10/dist-packages/hailo_sdk_client/allocator/hailo_tools_runner.py:179, in run_tool_from_binary(binary_path, tool_args, client, server, builder_pb_input)
    177 # Add the libraries for the or tools shared objects
    178 env = create_env()
--> 179 process = subprocess.Popen(cmd_args, env=env, pass_fds=(client.fileno(),))
    180 client.close()
    182 hef_data, output_proto_graph_bytes = server_loop(server, builder_pb_input=builder_pb_input)

File /usr/lib/python3.10/subprocess.py:971, in Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, user, group, extra_groups, encoding, errors, text, umask, pipesize)
    967         if self.text_mode:
    968             self.stderr = io.TextIOWrapper(self.stderr,
    969                     encoding=encoding, errors=errors)
--> 971     self._execute_child(args, executable, preexec_fn, close_fds,
    972                         pass_fds, cwd, env,
    973                         startupinfo, creationflags, shell,
    974                         p2cread, p2cwrite,
    975                         c2pread, c2pwrite,
    976                         errread, errwrite,
    977                         restore_signals,
    978                         gid, gids, uid, umask,
    979                         start_new_session)
    980 except:
    981     # Cleanup if the child failed starting.
    982     for f in filter(None, (self.stdin, self.stdout, self.stderr)):

File /usr/lib/python3.10/subprocess.py:1738, in Popen._execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session)
   1733     executable = args[0]
   1735 sys.audit("subprocess.Popen", executable, args, cwd, env)
   1737 if (_USE_POSIX_SPAWN
-> 1738         and os.path.dirname(executable)
   1739         and preexec_fn is None
   1740         and not close_fds
   1741         and not pass_fds
   1742         and cwd is None
   1743         and (p2cread == -1 or p2cread > 2)
   1744         and (c2pwrite == -1 or c2pwrite > 2)
   1745         and (errwrite == -1 or errwrite > 2)
   1746         and not start_new_session
   1747         and gid is None
   1748         and gids is None
   1749         and uid is None
   1750         and umask < 0):
   1751     self._posix_spawn(args, executable, env, restore_signals,
   1752                       p2cread, p2cwrite,
   1753                       c2pread, c2pwrite,
   1754                       errread, errwrite)
   1755     return

File /usr/lib/python3.10/posixpath.py:152, in dirname(p)
    150 def dirname(p):
    151     """Returns the directory component of a pathname"""
--> 152     p = os.fspath(p)
    153     sep = _get_sep(p)
    154     i = p.rfind(sep) + 1

TypeError: expected str, bytes or os.PathLike object, not NoneType

Any help?

I used the following parameters:

end_node_names=[‘/model.22/cv2.2/cv2.2.2/Conv’]

model_script_lines = [
“normalization1 = normalization([123.675, 116.28, 103.53], [58.395, 57.12, 57.375])\n”,
“model_optimization_flavor(optimization_level=2, batch_size=4)\n”,
#“quantization_param([conv42, conv53, conv63], force_range_out=[0.0, 1.0])\n”
]

Hi @wpo.eng17,

We usually see this error when a virtual env wasn’t used for the DFC. Is this the case here?

1 Like

It’s not like that.
I activate the environment first, and only then I run ‘hailo tutorial’ or start the jupyter notebook in the ‘home’ folder.

@wpo.eng17 you can try creating a new virtual environment and re-installing the DFC inside there. Let us know if the issue persists.

I’m still getting the same error. I re-did WSL with Ubuntu, created a new virtualenv, opened Jupyter Notebook with and without admin permission, and still got the same error

@nina-vilela I don’t know what else to do to make it work. Is it a bug in the DFC software?

Hi @wpo.eng17, what is the environment or PC that you are running this? Is this on the cloud?

2 Likes