How to use Linux Trace Toolkit: next generation (LTTng) to trace Hailo inference C++ code

[victorc]

LTTng (Linux Trace Toolkit: next generation) is a system software package for correlated tracing of the Linux kernel, applications and libraries.

LTTng uses the Tracepoint instrumentation of the Linux kernel, as well as various other information sources such as kprobes, and the Perf performance monitoring counters.

Designed for minimal performance impact and having a near-zero impact when not tracing, it is useful for debugging a wide range of bugs that are otherwise extremely challenging.

Here are the steps to apply LTTng to the tracing of Hailo inference code written in C++.

1. Follow the documentation to install LTTng: LTTng v2.13 — LTTng Documentation — LTTng
   If the installation is on Ubuntu 22.04, and if you got any error related to kernel 6.2 or 6.5 installation, please install the stable ppa version: LTTng v2.13 — LTTng Documentation — LTTng
2. Verify that the library liblttng-ust.so is in the expected path /usr/local/lib . If not, they may be in /usr/lib/x86_64-linux-gnu and you will have to add this path to the environment variable LD_LIBRARY_PATH
3. Install Trace Compass as trace visualizer: Trace Compass . After installation (basically uncompressing the tar.gz), try to run it by double-clicking on tracecompass icon and if you get an error related to incompatible Java JM version, please upgrade your Java Open JDK to OpenJDK 17 by following these instructions: How to Install Java on Ubuntu 22.04 | Step-by-Step Tutorial
4. To illustrate how to modify CPP application code to include tracing code, we are going to use the following yolov8 post estimation as example: Hailo-Application-Code-Examples/runtime/cpp/pose_estimation/yolov8_pose at main · hailo-ai/Hailo-Application-Code-Examples
   a) After cloning the repo, copy the attached files tp.c and tp.h in the directory …/Hailo-Application-Code-Examples/runtime/cpp/pose_estimation/yolov8_pose/ . These files define the tracepoint functions that the application will call. We are defining 3 functions to trace 3 different types of events:
   i. write event: when a new buffer is submitting to the input stream for inference
   ii. read event: when a processed buffer is obtained from the output vstream
   iii. control event: to alert when Hailo is initialized and released
   b) Modify the application code by calling the tracepoint functions at desired locations.
   You’ll also need to insert a line that prompts the user to press a key:

// Code to ask to press a key to continue
std::cout << BOLDBLUE << “-I- Press any key to continue” << RESET << std::endl;
std::cin.get();
Please see the attached file yolov8pose_example.cpp as example.

c) Modify the CMakeLists.txt by adding ./*o to the statement file(GLOB SOURCES
And after link_libraries(stdc++fs), add these 2 lines:
link_libraries(lttng-ust)
link_libraries(dl)
Please see attached example file CMAkeLists.txt.

5. Compile first the tracepoint files into a *.o file: gcc -c -I. tp.c
6. Build your application: ./build.sh
   If you are getting any linking error, verify your LD_LIBRARY_PATH variable has the path to the liblttng-ust.so library (/usr/local/lib or /usr/lib/x86_64-linux-gnu).
7. Run the application: ./build/x86_64/vstream_yolov8pose_example_cpp -hef=yolov8s_pose.hef -input=zidane.jpg -num=1000
   It will halt and ask you to press a key.
8. Open a separate terminal window and run:
   a) lttng-sessiond --daemonize
   b) lttng create my-user-space-session
   Pay attention to the message showing the directory name where the traces will be dumped
   Ex: Traces will be output to /home/victorc/lttng-traces/my-user-space-session-20240813-233621
   c) lttng enable-event --userspace hailo_yolov8_pose:control;lttng enable-event --userspace hailo_yolov8_pose:read;lttng enable-event --userspace hailo_yolov8_pose:write
   d) lttng start
9. Back to the other terminal window where your application is waiting for you keystroke, press a key to resume the application and the inference until completion.
10. Go back the terminal window where you executed ‘lttng start’ and run:
    lttng destroy
11. For visualization of the traces that were dumped in the trace directory, follow these instructions:
    a) Launch Trace Compass
    b) Select menu File->Open Trace
    c) Navigate to the directory where the traces were dumped. Go all the way down to the subdirectories containing the trace files 'channel’* and select any file. The software will load all the files in the directory and display the trace graph.
    d) Refer to the following doc to filter the events: Trace Compass User Guide - Trace Compass Main Features.
    e) Use the Colors menu to assign a color to a particular event. Use right-click to define the filter for the color as shown below:
    
    
    
    
    
    
    
    

[victorc]

Since the topic refers to some attached files and since it is not yet possible to attach files in this Community forum, here is content of these files.

tp.c

#define LTTNG_UST_TRACEPOINT_CREATE_PROBES

#define LTTNG_UST_TRACEPOINT_DEFINE

#include "tp.h"

tp.h

#undef LTTNG_UST_TRACEPOINT_PROVIDER
#define LTTNG_UST_TRACEPOINT_PROVIDER hailo_yolov8_pose

#undef LTTNG_UST_TRACEPOINT_INCLUDE
#define LTTNG_UST_TRACEPOINT_INCLUDE "./tp.h"

#if !defined(_HELLO_TP_H) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ)
#define _HELLO_TP_H

#include <lttng/tracepoint.h>

LTTNG_UST_TRACEPOINT_EVENT(
    hailo_yolov8_pose,
    read,
    LTTNG_UST_TP_ARGS(
        int, my_integer_arg,
        char *, my_string_arg
    ),
    LTTNG_UST_TP_FIELDS(
        lttng_ust_field_string(my_string_field, my_string_arg)
        lttng_ust_field_integer(int, my_integer_field, my_integer_arg)
    )
)

LTTNG_UST_TRACEPOINT_EVENT(
    hailo_yolov8_pose,
    write,
    LTTNG_UST_TP_ARGS(
        int, my_integer_arg,
        char *, my_string_arg
    ),
    LTTNG_UST_TP_FIELDS(
        lttng_ust_field_string(my_string_field, my_string_arg)
        lttng_ust_field_integer(int, my_integer_field, my_integer_arg)
    )
)

LTTNG_UST_TRACEPOINT_EVENT(
    hailo_yolov8_pose,
    control,
    LTTNG_UST_TP_ARGS(
        int, my_integer_arg,
        char *, my_string_arg
    ),
    LTTNG_UST_TP_FIELDS(
        lttng_ust_field_string(my_string_field, my_string_arg)
        lttng_ust_field_integer(int, my_integer_field, my_integer_arg)
    )
)

#endif /* _HELLO_TP_H */

#include <lttng/tracepoint-event.h>

yolov8pose_example.cpp

Add lttng_ust_tracepoint(hailo_yolov8_pose, read, i, name);
in read_all() function.
With char *name= (char*)output_vstream.get_info().name;

Add lttng_ust_tracepoint(hailo_yolov8_pose, write, i, (char*)"write single frame");
in use_single_frame() function.

Add lttng_ust_tracepoint(hailo_yolov8_pose, write, frame++, (char*)"write");
in write_all() function.

Add lttng_ust_tracepoint(hailo_yolov8_pose, control, 0, (char*)"begin");
at the beginning of main()

Add lttng_ust_tracepoint(hailo_yolov8_pose, control, 1, (char*)"end");
at the end of main()

CMakeLists.txt

cmake_minimum_required(VERSION 3.11)
project(vstream_yolov8pose_example_cpp)

set(CMAKE_CXX_STANDARD 20)

set(COMPILE_OPTIONS -Werror -Wall -Wextra -Wconversion -O3 -Wno-reorder -Wno-ignored-qualifiers -Wno-extra -Wno-unused-local-typedefs -Wno-conversion -Wno-parentheses -Wno-unused-but-set-variable -Wno-array-bounds -Wno-unused-value)

set(BASE_DIR /path/to/yolov8/example/folder)

set(CMAKE_THREAD_LIBS_INIT “-lpthread”)
SET(CMAKE_CXX_FLAGS “${CMAKE_CXX_FLAGS} -pthread”)
set(CMAKE_HAVE_THREADS_LIBRARY 1)
set(CMAKE_USE_WIN32_THREADS_INIT 0)
set(CMAKE_USE_PTHREADS_INIT 1)
set(THREADS_PREFER_PTHREAD_FLAG ON)

find_package(Threads)

find_package(HailoRT REQUIRED)

find_package( OpenCV REQUIRED)
message(STATUS "Found OpenCV: " ${OpenCV_INCLUDE_DIRS})

file(GLOB SOURCES
./.cpp
./.o
)

include(ExternalProject)

set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external)

ExternalProject_Add(xtl-test
GIT_REPOSITORY GitHub - xtensor-stack/xtl: The x template library
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION}
)

ExternalProject_Add(xtensor-test
GIT_REPOSITORY GitHub - xtensor-stack/xtensor: C++ tensors with broadcasting and lazy computing
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -Dxtl_DIR=${BASE_DIR}/build/x86_64/external/share/cmake/xtl/
)

include_directories(${EXTERNAL_INSTALL_LOCATION}/include)
link_directories(${EXTERNAL_INSTALL_LOCATION}/lib)

link_libraries(stdc++fs)
link_libraries(lttng-ust)
link_libraries(dl)
add_executable(${PROJECT_NAME} ${SOURCES})
add_dependencies(${PROJECT_NAME} xtl-test xtensor-test)
include_directories(${OpenCV_INCLUDE_DIRS})
include_directories(${ONNXRUNTIME_INCLUDE_DIR})
target_compile_options(${PROJECT_NAME} PRIVATE ${COMPILE_OPTIONS} -fconcepts)
target_link_libraries(${PROJECT_NAME} HailoRT::libhailort ${CMAKE_THREAD_LIBS_INIT} ${OpenCV_LIBS})