Hi,
I might have found a possible memleak in hailort.service running under yocto. Now I’m aware that hailort.service is not officially supported for yocto. I’d still appreciate any help since we’re dependent on running multiple models in parallel.
My setup includes a python script which continously restarts a gstreamer pipeline 10 times and sends EOS after 5 seconds. I track the memory of the gstreamer pipeline using memory_profiler as well as hailort.service.
- Test running hailonet without hailort.service
Pipeline:videotestsrc ! hailonet ! fakesink
No apparent memleak.
- Test running hailonet with hailort.service
Pipeline:videotestsrc ! hailonet multi-process-service=true vdevice-group-id=1 ! fakesink
And here is a valgrind report of hailort.service.
==137606== Memcheck, a memory error detector
==137606== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.
==137606== Using Valgrind-3.18.1 and LibVEX; rerun with -h for copyright info
==137606== Command: /usr/local/bin/hailort_service
==137606== Parent PID: 1
==137606==
==137606==
==137606== HEAP SUMMARY:
==137606== in use at exit: 52,084 bytes in 288 blocks
==137606== total heap usage: 552 allocs, 264 frees, 160,804 bytes allocated
==137606==
==137606== 320 bytes in 1 blocks are possibly lost in loss record 221 of 249
==137606== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137606== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137606== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137606== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137606== by 0x549119F: allocate_stack (allocatestack.c:428)
==137606== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137606== by 0x51EF367: std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (in /usr/lib/libstdc++.so.6.0.29)
==137606== by 0x4C19F0F: thread<spdlog::details::periodic_worker::periodic_worker(const std::function<void()>&, std::chrono::seconds)::<lambda()> > (std_thread.h:143)
==137606== by 0x4C19F0F: spdlog::details::periodic_worker::periodic_worker(std::function<void ()> const&, std::chrono::duration<long, std::ratio<1l, 1l> >) (periodic_worker-inl.h:21)
==137606== by 0x4BFC703: make_unique<spdlog::details::periodic_worker, spdlog::details::registry::flush_every(std::chrono::seconds)::<lambda()>&, std::chrono::duration<long int, std::ratio<1, 1> >&> (common.h:238)
==137606== by 0x4BFC703: spdlog::details::registry::flush_every(std::chrono::duration<long, std::ratio<1l, 1l> >) (registry-inl.h:191)
==137606== by 0x49909EF: hailort::HailoRTLogger::HailoRTLogger(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) (hailort_logger.cpp:201)
==137606== by 0x493C2AB: make_unique_nothrow<hailort::HailoRTLogger, spdlog::level::level_enum&, spdlog::level::level_enum&, spdlog::level::level_enum&> (utils.hpp:84)
==137606== by 0x493C2AB: hailort::HailoRTLogger::get_instance(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) [clone .isra.0] (hailort_logger.hpp:54)
==137606== by 0x493C4F3: hailort__initialize_logger (hailort.cpp:59)
==137606== by 0x493C4F3: hailort__initialize_logger_t_ (hailort.cpp:56)
==137606== by 0x493C4F3: __static_initialization_and_destruction_0 (hailort.cpp:56)
==137606== by 0x493C4F3: _GLOBAL__sub_I_hailort.cpp (hailort.cpp:2632)
==137606== by 0x4005787: call_init (dl-init.c:70)
==137606== by 0x4005787: call_init (dl-init.c:26)
==137606== by 0x400588B: _dl_init (dl-init.c:117)
==137606==
==137606== LEAK SUMMARY:
==137606== definitely lost: 0 bytes in 0 blocks
==137606== indirectly lost: 0 bytes in 0 blocks
==137606== possibly lost: 320 bytes in 1 blocks
==137606== still reachable: 51,764 bytes in 287 blocks
==137606== suppressed: 0 bytes in 0 blocks
==137606== Reachable blocks (those to which a pointer was found) are not shown.
==137606== To see them, rerun with: --leak-check=full --show-leak-kinds=all
==137606==
==137606== For lists of detected and suppressed errors, rerun with: -s
==137606== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 0 from 0)
==137814== Warning: noted but unhandled ioctl 0x6e02 with no size/direction hints.
==137814== This could cause spurious value errors to appear.
==137814== See README_MISSING_SYSCALL_OR_IOCTL for guidance on writing a proper wrapper.
==137814==
==137814== Process terminating with default action of signal 15 (SIGTERM)
==137814== at 0x54F3BE8: syscall (syscall.S:39)
==137814== by 0x5BE34F: absl::lts_20211102::synchronization_internal::Waiter::Wait(absl::lts_20211102::synchronization_internal::KernelTimeout) (in /usr/local/bin/hailort_service)
==137814== by 0x5BE20F: AbslInternalPerThreadSemWait_lts_20211102 (in /usr/local/bin/hailort_service)
==137814== by 0x5BDB87: absl::lts_20211102::CondVar::WaitCommon(absl::lts_20211102::Mutex*, absl::lts_20211102::synchronization_internal::KernelTimeout) (in /usr/local/bin/hailort_service)
==137814== by 0x33D74F: grpc::Server::Wait() (in /usr/local/bin/hailort_service)
==137814== by 0x2EE717: RunService() (in /usr/local/bin/hailort_service)
==137814== by 0x2AB13F: main (in /usr/local/bin/hailort_service)
==137814==
==137814== HEAP SUMMARY:
==137814== in use at exit: 232,282 bytes in 1,019 blocks
==137814== total heap usage: 267,258 allocs, 266,239 frees, 1,116,845,130 bytes allocated
==137814==
==137814== 56 bytes in 1 blocks are definitely lost in loss record 469 of 642
==137814== at 0x4865CFC: operator new(unsigned long) (vg_replace_malloc.c:422)
==137814== by 0x4C19E8B: thread<spdlog::details::periodic_worker::periodic_worker(const std::function<void()>&, std::chrono::seconds)::<lambda()> > (std_thread.h:143)
==137814== by 0x4C19E8B: spdlog::details::periodic_worker::periodic_worker(std::function<void ()> const&, std::chrono::duration<long, std::ratio<1l, 1l> >) (periodic_worker-inl.h:21)
==137814== by 0x4BFC703: make_unique<spdlog::details::periodic_worker, spdlog::details::registry::flush_every(std::chrono::seconds)::<lambda()>&, std::chrono::duration<long int, std::ratio<1, 1> >&> (common.h:238)
==137814== by 0x4BFC703: spdlog::details::registry::flush_every(std::chrono::duration<long, std::ratio<1l, 1l> >) (registry-inl.h:191)
==137814== by 0x49909EF: hailort::HailoRTLogger::HailoRTLogger(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) (hailort_logger.cpp:201)
==137814== by 0x493C2AB: make_unique_nothrow<hailort::HailoRTLogger, spdlog::level::level_enum&, spdlog::level::level_enum&, spdlog::level::level_enum&> (utils.hpp:84)
==137814== by 0x493C2AB: hailort::HailoRTLogger::get_instance(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) [clone .isra.0] (hailort_logger.hpp:54)
==137814== by 0x493C4F3: hailort__initialize_logger (hailort.cpp:59)
==137814== by 0x493C4F3: hailort__initialize_logger_t_ (hailort.cpp:56)
==137814== by 0x493C4F3: __static_initialization_and_destruction_0 (hailort.cpp:56)
==137814== by 0x493C4F3: _GLOBAL__sub_I_hailort.cpp (hailort.cpp:2632)
==137814== by 0x4005787: call_init (dl-init.c:70)
==137814== by 0x4005787: call_init (dl-init.c:26)
==137814== by 0x400588B: _dl_init (dl-init.c:117)
==137814== by 0x4017D57: ??? (in /lib/ld-linux-aarch64.so.1)
==137814==
==137814== 290 bytes in 10 blocks are definitely lost in loss record 572 of 642
==137814== at 0x486551C: malloc (vg_replace_malloc.c:381)
==137814== by 0x4B85B6B: hailort::TempFile::create(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (filesystem.cpp:228)
==137814== by 0x498AED3: hailort::MonitorHandler::open_temp_mon_file() (monitor_handler.cpp:222)
==137814== by 0x498B33B: hailort::MonitorHandler::start_mon(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (monitor_handler.cpp:186)
==137814== by 0x49C1A27: execute_trace<hailort::MonitorStartTrace, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > (tracer.hpp:53)
==137814== by 0x49C1A27: trace<hailort::MonitorStartTrace, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > (tracer.hpp:27)
==137814== by 0x49C1A27: hailort::VDeviceBase::create(hailo_vdevice_params_t const&) (vdevice.cpp:629)
==137814== by 0x49C51B3: operator() (vdevice.cpp:136)
==137814== by 0x49C51B3: register_resource<hailort::VDeviceHandle::create(const hailo_vdevice_params_t&)::<lambda()> > (shared_resource_manager.hpp:80)
==137814== by 0x49C51B3: hailort::VDeviceHandle::create(hailo_vdevice_params_t const&) (vdevice.cpp:138)
==137814== by 0x49C641B: hailort::VDevice::create(hailo_vdevice_params_t const&) (vdevice.cpp:555)
==137814== by 0x2C5D43: hailort::HailoRtRpcService::VDevice_create(grpc::ServerContext*, VDevice_create_Request const*, VDevice_create_Reply*) (in /usr/local/bin/hailort_service)
==137814== by 0x3628B3: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x39897B: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x34BE07: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x3515CF: grpc::ThreadManager::MainWorkLoop() (in /usr/local/bin/hailort_service)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 580 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x51EF367: std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (in /usr/lib/libstdc++.so.6.0.29)
==137814== by 0x4C19F0F: thread<spdlog::details::periodic_worker::periodic_worker(const std::function<void()>&, std::chrono::seconds)::<lambda()> > (std_thread.h:143)
==137814== by 0x4C19F0F: spdlog::details::periodic_worker::periodic_worker(std::function<void ()> const&, std::chrono::duration<long, std::ratio<1l, 1l> >) (periodic_worker-inl.h:21)
==137814== by 0x4BFC703: make_unique<spdlog::details::periodic_worker, spdlog::details::registry::flush_every(std::chrono::seconds)::<lambda()>&, std::chrono::duration<long int, std::ratio<1, 1> >&> (common.h:238)
==137814== by 0x4BFC703: spdlog::details::registry::flush_every(std::chrono::duration<long, std::ratio<1l, 1l> >) (registry-inl.h:191)
==137814== by 0x49909EF: hailort::HailoRTLogger::HailoRTLogger(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) (hailort_logger.cpp:201)
==137814== by 0x493C2AB: make_unique_nothrow<hailort::HailoRTLogger, spdlog::level::level_enum&, spdlog::level::level_enum&, spdlog::level::level_enum&> (utils.hpp:84)
==137814== by 0x493C2AB: hailort::HailoRTLogger::get_instance(spdlog::level::level_enum, spdlog::level::level_enum, spdlog::level::level_enum) [clone .isra.0] (hailort_logger.hpp:54)
==137814== by 0x493C4F3: hailort__initialize_logger (hailort.cpp:59)
==137814== by 0x493C4F3: hailort__initialize_logger_t_ (hailort.cpp:56)
==137814== by 0x493C4F3: __static_initialization_and_destruction_0 (hailort.cpp:56)
==137814== by 0x493C4F3: _GLOBAL__sub_I_hailort.cpp (hailort.cpp:2632)
==137814== by 0x4005787: call_init (dl-init.c:70)
==137814== by 0x4005787: call_init (dl-init.c:26)
==137814== by 0x400588B: _dl_init (dl-init.c:117)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 581 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x5906BB: grpc_core::Thread::Thread(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&) (in /usr/local/bin/hailort_service)
==137814== by 0x4963AF: grpc_core::Executor::SetThreading(bool) (in /usr/local/bin/hailort_service)
==137814== by 0x496E5F: grpc_core::Executor::InitAll() (in /usr/local/bin/hailort_service)
==137814== by 0x558DF7: grpc_iomgr_init() (in /usr/local/bin/hailort_service)
==137814== by 0x4CFB53: grpc_init (in /usr/local/bin/hailort_service)
==137814== by 0x350AD3: grpc::ServerCredentials::ServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x3392AF: grpc::InsecureServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x2EE6B7: RunService() (in /usr/local/bin/hailort_service)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 582 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x5906BB: grpc_core::Thread::Thread(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&) (in /usr/local/bin/hailort_service)
==137814== by 0x4963AF: grpc_core::Executor::SetThreading(bool) (in /usr/local/bin/hailort_service)
==137814== by 0x496E67: grpc_core::Executor::InitAll() (in /usr/local/bin/hailort_service)
==137814== by 0x558DF7: grpc_iomgr_init() (in /usr/local/bin/hailort_service)
==137814== by 0x4CFB53: grpc_init (in /usr/local/bin/hailort_service)
==137814== by 0x350AD3: grpc::ServerCredentials::ServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x3392AF: grpc::InsecureServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x2EE6B7: RunService() (in /usr/local/bin/hailort_service)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 583 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x5906BB: grpc_core::Thread::Thread(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&) (in /usr/local/bin/hailort_service)
==137814== by 0x4A2727: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x4CFB97: grpc_init (in /usr/local/bin/hailort_service)
==137814== by 0x350AD3: grpc::ServerCredentials::ServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x3392AF: grpc::InsecureServerCredentials() (in /usr/local/bin/hailort_service)
==137814== by 0x2EE6B7: RunService() (in /usr/local/bin/hailort_service)
==137814== by 0x2AB13F: main (in /usr/local/bin/hailort_service)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 584 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x5906BB: grpc_core::Thread::Thread(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&) (in /usr/local/bin/hailort_service)
==137814== by 0x4A2727: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x4A3073: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x5904EF: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x54906F7: start_thread (pthread_create.c:442)
==137814== by 0x54F7CDB: thread_start (clone.S:79)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 585 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x51EF367: std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (in /usr/lib/libstdc++.so.6.0.29)
==137814== by 0x498B40F: thread<hailort::MonitorHandler::start_mon(const string&)::<lambda()> > (std_thread.h:143)
==137814== by 0x498B40F: hailort::MonitorHandler::start_mon(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (monitor_handler.cpp:190)
==137814== by 0x49C1A27: execute_trace<hailort::MonitorStartTrace, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > (tracer.hpp:53)
==137814== by 0x49C1A27: trace<hailort::MonitorStartTrace, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > (tracer.hpp:27)
==137814== by 0x49C1A27: hailort::VDeviceBase::create(hailo_vdevice_params_t const&) (vdevice.cpp:629)
==137814== by 0x49C51B3: operator() (vdevice.cpp:136)
==137814== by 0x49C51B3: register_resource<hailort::VDeviceHandle::create(const hailo_vdevice_params_t&)::<lambda()> > (shared_resource_manager.hpp:80)
==137814== by 0x49C51B3: hailort::VDeviceHandle::create(hailo_vdevice_params_t const&) (vdevice.cpp:138)
==137814== by 0x49C641B: hailort::VDevice::create(hailo_vdevice_params_t const&) (vdevice.cpp:555)
==137814== by 0x2C5D43: hailort::HailoRtRpcService::VDevice_create(grpc::ServerContext*, VDevice_create_Request const*, VDevice_create_Reply*) (in /usr/local/bin/hailort_service)
==137814== by 0x3628B3: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x39897B: ??? (in /usr/local/bin/hailort_service)
==137814==
==137814== 320 bytes in 1 blocks are possibly lost in loss record 586 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x51EF367: std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (in /usr/lib/libstdc++.so.6.0.29)
==137814== by 0x4A37077: thread<hailort::vdma::TransferLauncher::TransferLauncher()::<lambda()> > (std_thread.h:143)
==137814== by 0x4A37077: hailort::vdma::TransferLauncher::TransferLauncher() (transfer_launcher.cpp:30)
==137814== by 0x4A37BBF: make_unique_nothrow<hailort::vdma::TransferLauncher> (utils.hpp:84)
==137814== by 0x4A37BBF: hailort::vdma::TransferLauncher::create() (transfer_launcher.cpp:19)
==137814== by 0x4A1EAAF: hailort::VdmaDevice::add_hef(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (vdma_device.cpp:157)
==137814== by 0x49A84D7: hailort::DeviceBase::configure(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (device_internal.cpp:54)
==137814== by 0x49C05DB: hailort::VDeviceBase::create_physical_core_op(hailort::Device&, hailort::Hef&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, hailort::ConfigureNetworkParams const&) (vdevice.cpp:900)
==137814== by 0x49C0B13: hailort::VDeviceBase::create_vdevice_core_op(hailort::Hef&, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> const&) (vdevice.cpp:928)
==137814== by 0x49C2FC7: hailort::VDeviceBase::configure(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (vdevice.cpp:718)
==137814==
==137814== 640 bytes in 2 blocks are possibly lost in loss record 594 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x5906BB: grpc_core::Thread::Thread(char const*, void (*)(void*), void*, bool*, grpc_core::Thread::Options const&) (in /usr/local/bin/hailort_service)
==137814== by 0x350D67: grpc::ThreadManager::WorkerThread::WorkerThread(grpc::ThreadManager*) (in /usr/local/bin/hailort_service)
==137814== by 0x351693: grpc::ThreadManager::MainWorkLoop() (in /usr/local/bin/hailort_service)
==137814== by 0x3517DB: grpc::ThreadManager::WorkerThread::Run() (in /usr/local/bin/hailort_service)
==137814== by 0x5904EF: ??? (in /usr/local/bin/hailort_service)
==137814== by 0x54906F7: start_thread (pthread_create.c:442)
==137814== by 0x54F7CDB: thread_start (clone.S:79)
==137814==
==137814== 1,600 bytes in 5 blocks are possibly lost in loss record 629 of 642
==137814== at 0x486A790: calloc (vg_replace_malloc.c:1328)
==137814== by 0x4010DD7: calloc (rtld-malloc.h:44)
==137814== by 0x4010DD7: allocate_dtv (dl-tls.c:375)
==137814== by 0x4011843: _dl_allocate_tls (dl-tls.c:634)
==137814== by 0x549119F: allocate_stack (allocatestack.c:428)
==137814== by 0x549119F: pthread_create@@GLIBC_2.34 (pthread_create.c:647)
==137814== by 0x51EF367: std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) (in /usr/lib/libstdc++.so.6.0.29)
==137814== by 0x4A360F7: thread<hailort::vdma::InterruptsDispatcher::InterruptsDispatcher(std::reference_wrapper<hailort::HailoRTDriver>)::<lambda()> > (std_thread.h:143)
==137814== by 0x4A360F7: hailort::vdma::InterruptsDispatcher::InterruptsDispatcher(std::reference_wrapper<hailort::HailoRTDriver>) (interrupts_dispatcher.cpp:26)
==137814== by 0x4A36B5B: make_unique_nothrow<hailort::vdma::InterruptsDispatcher, std::reference_wrapper<hailort::HailoRTDriver>&> (utils.hpp:84)
==137814== by 0x4A36B5B: hailort::vdma::InterruptsDispatcher::create(std::reference_wrapper<hailort::HailoRTDriver>) (interrupts_dispatcher.cpp:19)
==137814== by 0x4A1EA6B: hailort::VdmaDevice::add_hef(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (vdma_device.cpp:154)
==137814== by 0x49A84D7: hailort::DeviceBase::configure(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (device_internal.cpp:54)
==137814== by 0x49C05DB: hailort::VDeviceBase::create_physical_core_op(hailort::Device&, hailort::Hef&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, hailort::ConfigureNetworkParams const&) (vdevice.cpp:900)
==137814== by 0x49C0B13: hailort::VDeviceBase::create_vdevice_core_op(hailort::Hef&, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> const&) (vdevice.cpp:928)
==137814== by 0x49C2FC7: hailort::VDeviceBase::configure(hailort::Hef&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, hailort::ConfigureNetworkParams, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, hailort::ConfigureNetworkParams> > > const&) (vdevice.cpp:718)
==137814==
==137814== 2,304 bytes in 1 blocks are possibly lost in loss record 630 of 642
==137814== at 0x486551C: malloc (vg_replace_malloc.c:381)
==137814== by 0x400504B: malloc (rtld-malloc.h:56)
==137814== by 0x400504B: _dlfo_mappings_segment_allocate (dl-find_object.c:217)
==137814== by 0x400504B: _dl_find_object_update_1 (dl-find_object.c:671)
==137814== by 0x400504B: _dl_find_object_update (dl-find_object.c:805)
==137814== by 0x400C923: dl_open_worker_begin (dl-open.c:735)
==137814== by 0x553D4E7: _dl_catch_exception (dl-error-skeleton.c:208)
==137814== by 0x400BD97: dl_open_worker (dl-open.c:782)
==137814== by 0x553D4E7: _dl_catch_exception (dl-error-skeleton.c:208)
==137814== by 0x400C19B: _dl_open (dl-open.c:884)
==137814== by 0x553D86F: do_dlopen (dl-libc.c:95)
==137814== by 0x553D4E7: _dl_catch_exception (dl-error-skeleton.c:208)
==137814== by 0x553D59F: _dl_catch_error (dl-error-skeleton.c:227)
==137814== by 0x553D7C7: dlerror_run (dl-libc.c:45)
==137814== by 0x553D9CB: __libc_dlopen_mode (dl-libc.c:162)
==137814==
==137814== LEAK SUMMARY:
==137814== definitely lost: 346 bytes in 11 blocks
==137814== indirectly lost: 0 bytes in 0 blocks
==137814== possibly lost: 6,784 bytes in 15 blocks
==137814== still reachable: 225,152 bytes in 993 blocks
==137814== suppressed: 0 bytes in 0 blocks
==137814== Reachable blocks (those to which a pointer was found) are not shown.
==137814== To see them, rerun with: --leak-check=full --show-leak-kinds=all
==137814==
==137814== For lists of detected and suppressed errors, rerun with: -s
==137814== ERROR SUMMARY: 12 errors from 12 contexts (suppressed: 0 from 0)
And here’s the script I’m using to run the pipeline
import gi
import time
gi.require_version('Gst', '1.0')
gi.require_version('GstBase', '1.0')
gi.require_version('GObject', '2.0')
from gi.repository import Gst, GLib, GObject
import argparse
from pathlib import Path
from multiprocessing import Pool
from dataclasses import dataclass
import gc
@dataclass
class Args:
n_restart: int
input_file: Path
eos_after: int
port: int
def parser_init():
parser = argparse.ArgumentParser(description='Count People')
parser.add_argument("--input",
help="path to src video",
action="store",
dest="src_video",
required=True,
type=Path)
parser.add_argument("--eos_after",
help="Send EOS after n seconds",
action="store",
dest="eos_after",
type=int)
parser.add_argument("--parallel",
help="N of pipelines running in parallel",
action="store",
dest="n_parallel",
default=1,
type=int)
parser.add_argument("--n_restart",
help="N of restarts per pipeline",
action="store",
dest="n_restart",
default=1,
type=int)
return parser
def check_bus(args):
bus, mainloop = args
msg = bus.pop()
if msg and msg.type == Gst.MessageType.EOS:
mainloop.quit()
return False
return True
def send_eos(args):
pipeline, port = args
print(f"Sending EOS to the pipeline {port}...")
pipeline.send_event(Gst.Event.new_eos())
return False
def worker(p_args: Args):
Gst.init(None)
for i in range(p_args.n_restart):
print(f"pipeline {p_args.port} run {i}")
videotestsrc = Gst.ElementFactory.make("videotestsrc", "videotestsrc")
hailonet = Gst.ElementFactory.make("hailonet", "hailonet")
fakesink = Gst.ElementFactory.make("fakesink", "fakesink")
hailonet.set_property("hef-path", "model/yolov7_tiny.hef")
hailonet.set_property("multi-process-service", True)
hailonet.set_property("vdevice-group-id", 1)
pipeline = Gst.Pipeline.new(f"pipeline-{p_args.port}")
pipeline.add(videotestsrc)
pipeline.add(hailonet)
pipeline.add(fakesink)
videotestsrc.link(hailonet)
hailonet.link(fakesink)
ret = pipeline.set_state(Gst.State.PLAYING)
if ret == Gst.StateChangeReturn.FAILURE:
print("Unable to set the pipeline to the playing state.")
exit(-1)
mainloop = GLib.MainLoop()
bus = pipeline.get_bus()
if p_args.eos_after is not None:
GLib.timeout_add_seconds(p_args.eos_after, send_eos, [pipeline, p_args.port])
GLib.timeout_add_seconds(0.1, check_bus, [bus, mainloop])
try:
mainloop.run()
except KeyboardInterrupt:
pass
ret = pipeline.set_state(Gst.State.NULL)
assert(ret == Gst.StateChangeReturn.SUCCESS)
del pipeline
del bus
del mainloop
del videotestsrc
del hailonet
del fakesink
gc.collect()
if __name__ == "__main__":
args = vars(parser_init().parse_args())
args_processes = []
port = 5555
for _ in range(args["n_parallel"]):
args_processes.append(Args(n_restart=args["n_restart"], input_file=args["src_video"], eos_after=args["eos_after"], port=port))
port += 1
with Pool(processes=args["n_parallel"]) as pool:
pool.map(worker, args_processes)
Versions and Environment:
imx8m Plus
hailo-firmware - 4.18.0-r0.0
hailo-pci - 4.18.0-r0.0
hailo-post-processes - 3.29.0-r0.0
hailortcli - 4.18.0-r0.0
hailortservice - 4.18.0-r0.0
kernel-module-hailo-pci-5.15.71-5.15.71-2.2.0+g3ac48086a5b6 - 4.18.0-r0.0
libgsthailo - 4.18.0-r0.0
libgsthailotools - 3.29.0-r0.1
libhailort4.18.0 - 4.18.0-r0.0