I got it working, I had to to also reset the network group when unloading a model.
#include "hailo/hailort.hpp"
#include <iostream>
#include <unistd.h>
#define HEF_FILE ("/tmp/ai_models/yolov4/tiny_yolov4_license_plates.hef")
constexpr size_t FRAMES_COUNT = 1;
constexpr hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO;
std::unique_ptr<hailort::Hef> hef;
std::shared_ptr<hailort::ConfiguredNetworkGroup> network_group;
std::unique_ptr<hailort::InferVStreams> pipeline;
std::map<std::string, hailort::MemoryView> input_data_mem_view;
std::map<std::string, hailort::MemoryView> output_data_mem_view;
#define INPUT_TOT_SIZE (416 * 416 * 3)
#define OUTPUT_TOT_SIZE (1014 + 507 + 507 + 1014 + 4056 + 2028 + 2028 + 4056)
int sums[5] = {0};
using namespace hailort;
std::unique_ptr<hailort::VDevice> vDevice;
// Cumulative sum of elements of given array
uint32_t sumArray(const uint8_t *array, size_t size)
{
uint32_t sum = 0;
for (size_t i = 0; i < size; ++i)
{
sum += array[i];
}
return sum;
}
int input_data_yolov4(uint8_t *data)
{
const char *filename = "/tmp/ai_models/yolov4/enzo.txt";
int x = 0;
FILE *file = fopen(filename, "r");
if (!file)
{
printf("Error opening file`%s` input_data_yolov4", filename);
return -1;
}
// Calculate the expected size
size_t expected_size = 416 * 416 * 3;
size_t size = 0; // Initialize the size output parameter
// Read data from the file
while (fscanf(file, "%hhd", (uint8_t *)&data[size]) == 1)
{
// printf("%ld - \n", size);
++(size);
if (size > expected_size)
{
fprintf(stderr, "Error: Transfer is incomplete, too much data.\n");
fclose(file);
return -1;
}
}
fclose(file);
return 0;
}
Expected<std::shared_ptr<ConfiguredNetworkGroup>> configure_network_group(VDevice &vdevice)
{
auto hef_exp = Hef::create(HEF_FILE);
if (!hef_exp)
{
return make_unexpected(hef_exp.status());
}
hef = std::make_unique<hailort::Hef>(hef_exp.release());
auto configure_params = vdevice.create_configure_params(*hef);
if (!configure_params)
{
return make_unexpected(configure_params.status());
}
auto network_groups = vdevice.configure(*hef, configure_params.value());
if (!network_groups)
{
return make_unexpected(network_groups.status());
}
if (1 != network_groups->size())
{
std::cerr << "Invalid amount of network groups" << std::endl;
return make_unexpected(HAILO_INTERNAL_FAILURE);
}
return std::move(network_groups->at(0));
}
// Creates pipeline object and input/output MemoryView maps given a vDevice
int setup(uint8_t *in, uint8_t *out)
{
auto network_group_exp = configure_network_group(*vDevice);
if (!network_group_exp)
{
std::cerr << "Failed to configure network group " << HEF_FILE << std::endl;
return network_group_exp.status();
}
network_group = network_group_exp.release();
auto input_params = network_group->make_input_vstream_params({}, FORMAT_TYPE, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE);
if (!input_params)
{
std::cerr << "Failed make_input_vstream_params " << input_params.status() << std::endl;
return input_params.status();
}
auto output_params = network_group->make_output_vstream_params({}, FORMAT_TYPE, HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE);
if (!output_params)
{
std::cerr << "Failed make_output_vstream_params " << output_params.status() << std::endl;
return output_params.status();
}
auto pipeline_exp = InferVStreams::create(*network_group, input_params.value(), output_params.value());
if (!pipeline_exp)
{
std::cerr << "Failed to create inference pipeline " << pipeline_exp.status() << std::endl;
return pipeline_exp.status();
}
pipeline = std::make_unique<hailort::InferVStreams>(std::move(pipeline_exp.release()));
void *ptr = nullptr;
// Input streams processing
// Fill out `input_data_mem_view`
int current_input_vector = 0;
int cumulative_size = 0;
for (const auto &input_vstream : pipeline->get_input_vstreams())
{
ptr = (void *)&(in[cumulative_size]);
auto frame_size = input_vstream.get().get_frame_size();
input_data_mem_view.emplace(input_vstream.get().name(), hailort::MemoryView(ptr, frame_size));
current_input_vector++;
cumulative_size += frame_size;
}
// Output streams processing
// Fill out `output_data_mem_view`
int current_output_vector = 0;
cumulative_size = 0;
for (const auto &output_vstream : pipeline->get_output_vstreams())
{
ptr = (void *)&(out[cumulative_size]);
auto frame_size = output_vstream.get().get_frame_size();
output_data_mem_view.emplace(output_vstream.get().name(), hailort::MemoryView(ptr, frame_size));
current_output_vector++;
cumulative_size += frame_size;
}
return 0;
}
void printVectors(uint8_t *input_v, uint8_t *output_v)
{
sums[0] = sumArray(input_v, 416 * 416 * 3);
sums[1] = sumArray(output_v + 0, 1014);
sums[2] = sumArray(output_v + 1014, 507);
sums[3] = sumArray(output_v + 1014 + 507, 507);
sums[4] = sumArray(output_v + 1014 + 507 + 507, 1014);
printf("%d-%d-%d-%d-%d\n", sums[0], sums[1], sums[2], sums[3], sums[4]);
}
int main()
{
uint8_t *input_v = (uint8_t *)malloc(INPUT_TOT_SIZE);
uint8_t *output_v = (uint8_t *)malloc(OUTPUT_TOT_SIZE);
input_data_yolov4(input_v);
// Create a vDevice instance only once
auto tmp_vDevice = VDevice::create();
if (!tmp_vDevice)
{
std::cerr << "Failed to create vdevice, status = " << tmp_vDevice.status() << std::endl;
return tmp_vDevice.status();
}
vDevice = std::move(tmp_vDevice.value());
int load_unload_cycles = 0;
int inference_cycles = 0;
while (load_unload_cycles < 100)
{
std::cout << "Round: " << load_unload_cycles << std::endl;
setup(input_v, output_v);
inference_cycles = 0;
while (inference_cycles < 10)
{
// Zero out output vectors
std::memset(output_v + 0, 0, 1014);
std::memset(output_v + 1014, 0, 507);
std::memset(output_v + 1014 + 507, 0, 507);
std::memset(output_v + 1014 + 507 + 507, 0, 1014);
std::cout << "Before : ";
printVectors(input_v, output_v);
auto status = pipeline->infer(input_data_mem_view, output_data_mem_view, 1);
if (HAILO_SUCCESS != status)
{
std::cout << "Inference error: " << status << std::endl;
}
std::cout << "After : ";
printVectors(input_v, output_v);
inference_cycles++;
std::cout << std::endl;
}
std::cout << "-----" << std::endl;
load_unload_cycles++;
pipeline.reset();
input_data_mem_view.clear();
output_data_mem_view.clear();
network_group.reset();
hef.reset();
}
vDevice.reset();
free(input_v);
free(output_v);
return 0;
}