HI -
I’m trying to deploy this: Falconsai/nsfw_image_detection · Hugging Face
On the Hailo 8.
This verification code works fine on the quantized model .har.
import numpy as np
from PIL import Image
from hailo_sdk_client import ClientRunner
from hailo_sdk_client.exposed_definitions import InferenceContext
import os
import argparse
import time
import psutil
import gc
def log_memory_usage(stage):
process = psutil.Process(os.getpid())
print(f"\nMemory usage at {stage}: {process.memory_info().rss / 1024 / 1024:.2f} MB")
def simple_verify_har(img_path, har_path):
"""Simple HAR model verification with performance monitoring"""
start_total = time.time()
# Track initialization time
print("Initializing ClientRunner...")
log_memory_usage("before initialization")
t0 = time.time()
runner = ClientRunner(hef=har=har_path)
print(f"Initialization took: {time.time() - t0:.2f} seconds")
log_memory_usage("after initialization")
# Track image loading and preprocessing time
print("\nLoading and preprocessing image...")
t0 = time.time()
img = Image.open(img_path).convert('RGB')
img = img.resize((224, 224))
img_array = np.array(img).astype(np.float32) / 255.0
img_array = np.expand_dims(img_array, 0)
print(f"Image preprocessing took: {time.time() - t0:.2f} seconds")
log_memory_usage("after preprocessing")
# Get input layer info
print("\nGetting model information...")
t0 = time.time()
input_layer = [layer for layer in runner.get_hn_dict()["layers"]
if runner.get_hn_dict()["layers"][layer]["type"] == "input_layer"][0]
print(f"Getting model info took: {time.time() - t0:.2f} seconds")
# Track inference time
print("\nRunning inference...")
t0 = time.time()
with runner.infer_context(InferenceContext.SDK_NATIVE) as ctx:
print("Context created, starting inference...")
t1 = time.time()
outputs = runner.infer(ctx, {input_layer: img_array}, batch_size=1)
print(f"Pure inference took: {time.time() - t1:.2f} seconds")
print(f"Total inference context took: {time.time() - t0:.2f} seconds")
log_memory_usage("after inference")
# Process results
probabilities = softmax(outputs[0])
print("\nResults:")
print(f"Safe probability: {probabilities[0]:.4f}")
print(f"NSFW probability: {probabilities[1]:.4f}")
print(f"\nTotal execution time: {time.time() - start_total:.2f} seconds")
def softmax(x):
exp_x = np.exp(x - np.max(x))
return exp_x / exp_x.sum()
def main():
parser = argparse.ArgumentParser(description='Run inference on HAR model with diagnostics')
parser.add_argument('--image', required=True, help='Path to input image')
parser.add_argument('--har', required=True, help='Path to HAR file')
args = parser.parse_args()
# Clear any existing memory
gc.collect()
simple_verify_har(args.image, args.har)
if __name__ == "__main__":
main()
Then I run hailo compiler nsfw_model_quantized.har
This works, no errors…
Then I run the verification script on the saved .har file AFTER the compilation is complete.
I know the script is loading the compiled model because the debug logs show the size as being much smaller.
The results are correct stlll…
Then I run inference on the .hef using the HailoRT library, and I get basically 50/50 results on everything. However, I’m not sure how to debug the issue? Can anyone help?
detector.cpp
#include "hailo/hailort.hpp"
#include <opencv2/opencv.hpp>
#include <iostream>
#include <vector>
#include <chrono>
#include <iomanip>
using namespace hailort;
class NSFWDetector {
private:
std::unique_ptr<VDevice> vdevice;
std::shared_ptr<ConfiguredNetworkGroup> network_group;
std::unique_ptr<InferVStreams> pipeline;
std::map<std::string, std::vector<uint8_t>> input_data;
std::map<std::string, std::vector<uint8_t>> output_data;
std::map<std::string, MemoryView> input_views;
std::map<std::string, MemoryView> output_views;
const std::string HEF_FILE = "nsfw_model.hef";
constexpr static hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO;
Expected<std::shared_ptr<ConfiguredNetworkGroup>> configure_network_group(VDevice &vdevice) {
auto hef = Hef::create(HEF_FILE);
if (!hef) return make_unexpected(hef.status());
auto configure_params = vdevice.create_configure_params(hef.value());
if (!configure_params) return make_unexpected(configure_params.status());
auto network_groups = vdevice.configure(hef.value(), configure_params.value());
if (!network_groups) return make_unexpected(network_groups.status());
if (1 != network_groups->size()) return make_unexpected(HAILO_INTERNAL_FAILURE);
return std::move(network_groups->at(0));
}
void print_vstream_info() {
std::cout << "\n=== Input VStreams (" << pipeline->get_input_vstreams().size() << ") ===" << std::endl;
for (const auto &input_vstream : pipeline->get_input_vstreams()) {
auto info = input_vstream.get().get_info();
std::cout << "\nVStream Name: " << input_vstream.get().name() << std::endl;
std::cout << "Format:" << std::endl;
std::cout << " Type: " << info.format.type << std::endl;
std::cout << " Order: " << info.format.order << std::endl;
std::cout << "Shape:" << std::endl;
std::cout << " Height: " << info.shape.height << std::endl;
std::cout << " Width: " << info.shape.width << std::endl;
std::cout << " Features: " << info.shape.features << std::endl;
std::cout << "Frame Size: " << input_vstream.get().get_frame_size() << " bytes" << std::endl;
}
}
std::pair<float, float> calculate_probabilities(const uint8_t* scores) {
// Convert uint8 scores to probabilities using softmax
float score0 = static_cast<float>(scores[0]);
float score1 = static_cast<float>(scores[1]);
// Apply softmax
float max_score = std::max(score0, score1);
float exp0 = std::exp(score0 - max_score);
float exp1 = std::exp(score1 - max_score);
float sum = exp0 + exp1;
return std::make_pair(exp0/sum, exp1/sum);
}
public:
hailo_status init() {
auto vdevice_exp = VDevice::create();
if (!vdevice_exp) return vdevice_exp.status();
vdevice = std::unique_ptr<VDevice>(vdevice_exp.release());
auto network_group_exp = configure_network_group(*vdevice);
if (!network_group_exp) return network_group_exp.status();
network_group = network_group_exp.value();
auto input_params = network_group->make_input_vstream_params({}, FORMAT_TYPE,
HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE);
if (!input_params) return input_params.status();
auto output_params = network_group->make_output_vstream_params({}, FORMAT_TYPE,
HAILO_DEFAULT_VSTREAM_TIMEOUT_MS, HAILO_DEFAULT_VSTREAM_QUEUE_SIZE);
if (!output_params) return output_params.status();
auto pipeline_exp = InferVStreams::create(*network_group, input_params.value(), output_params.value());
if (!pipeline_exp) return pipeline_exp.status();
pipeline = std::make_unique<InferVStreams>(std::move(pipeline_exp.value()));
print_vstream_info();
// Pre-allocate buffers
for (const auto &input_vstream : pipeline->get_input_vstreams()) {
size_t frame_size = input_vstream.get().get_frame_size();
input_data.emplace(input_vstream.get().name(), std::vector<uint8_t>(frame_size));
input_views.emplace(input_vstream.get().name(),
MemoryView(input_data[input_vstream.get().name()].data(),
input_data[input_vstream.get().name()].size()));
}
for (const auto &output_vstream : pipeline->get_output_vstreams()) {
size_t frame_size = output_vstream.get().get_frame_size();
output_data.emplace(output_vstream.get().name(), std::vector<uint8_t>(frame_size));
output_views.emplace(output_vstream.get().name(),
MemoryView(output_data[output_vstream.get().name()].data(),
output_data[output_vstream.get().name()].size()));
}
return HAILO_SUCCESS;
}
hailo_status detect(const cv::Mat& image) {
auto start_time = std::chrono::high_resolution_clock::now();
for (const auto &input_vstream : pipeline->get_input_vstreams()) {
// 1. Get shape info
auto shape = input_vstream.get().get_info().shape;
// 2. Resize image
cv::Mat resized;
cv::resize(image, resized, cv::Size(shape.width, shape.height));
// 3. Convert BGR to RGB since model expects RGB
cv::Mat rgb_image;
cv::cvtColor(resized, rgb_image, cv::COLOR_BGR2RGB);
// 4. Convert to float and normalize to 0-1
cv::Mat float_img;
rgb_image.convertTo(float_img, CV_32F, 1.0/255.0);
// 5. Convert back to uint8 for the hardware
cv::Mat uint8_img;
float_img.convertTo(uint8_img, CV_8U, 255.0);
// Debug print first few values
std::cout << "Input values: ";
for(int i = 0; i < 5; i++) {
std::cout << (int)uint8_img.data[i] << " ";
}
std::cout << std::endl;
// Copy to input buffer
std::memcpy(input_data[input_vstream.get().name()].data(),
uint8_img.data,
input_data[input_vstream.get().name()].size());
}
auto status = pipeline->infer(input_views, output_views, 1);
if (status == HAILO_SUCCESS) {
for (const auto &output_vstream : pipeline->get_output_vstreams()) {
const auto& output = output_data[output_vstream.get().name()];
const uint8_t* scores = output.data();
// Print raw scores for debugging
std::cout << "Raw scores: " << (int)scores[0] << ", " << (int)scores[1] << std::endl;
// Calculate probabilities
auto [prob_safe, prob_nsfw] = calculate_probabilities(scores);
std::cout << "\nResults:" << std::endl;
std::cout << "Safe: " << std::fixed << std::setprecision(4)
<< prob_safe * 100 << "%" << std::endl;
std::cout << "NSFW: " << std::fixed << std::setprecision(4)
<< prob_nsfw * 100 << "%" << std::endl;
}
}
auto end_time = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
std::cout << "Inference time: " << duration.count() << "ms" << std::endl;
return status;
}
};
int main(int argc, char** argv) {
if (argc != 2) {
std::cerr << "Usage: " << argv[0] << " <image_path>" << std::endl;
return -1;
}
std::string image_path = argv[1];
cv::Mat image = cv::imread(image_path);
if (image.empty()) {
std::cerr << "Failed to load image: " << image_path << std::endl;
return -1;
}
NSFWDetector detector;
auto status = detector.init();
if (HAILO_SUCCESS != status) {
std::cerr << "Failed to initialize detector" << std::endl;
return status;
}
status = detector.detect(image);
if (HAILO_SUCCESS != status) {
std::cerr << "Failed to run inference" << std::endl;
return status;
}
return 0;
}