Hi i geting ERRor Negative Slope ExponentNonFixable

[vinaygouda.ttssl]

Negative Slope ExponentNonFixable: Quantization failed in layer custom_violation_model_700epoch_v2/conv83 due to unsupported required slope. Desired shift is 19.0, but op has only 8 data bits. This error raises when the data or weight range are not balanced. Mostly happens when using random calibration-set/weights, the calibration-set is not normalized properly or batch-normalization was not used during training. @Nadav @omria @Omer

[vinaygouda.ttssl]

plz any one tell me why this error ocured an how to solve it

[nina-vilela]

Hi @vinaygouda.ttssl,

As the error message says, this issue usually happens when there are issues with normalization/calibset/weights. Could you share the following details with us:

1. Model architecture
2. Model script used
3. How calibset was created

[vinaygouda.ttssl]

Model architecture yolov8
Calibration set has been created as per the hailo tutorials

[nina-vilela]

please try this solution:

Problem With Model Optimization

I might have a solution, but I’m not sure if it’s general, but I think that it worth checking. It seems that on datasets that have low number of classes and/or low number of pictures, some nodes on the YOLOv8 are getting almost nullified. I’ve used this command to force a wider dynamic range on the outputs: quantization_param([conv42, conv53, conv63], force_range_out=[0.0, 1.0]) This is relevant for YOLOV8s trained from our retraining dockers. These are the end nodes before the NMS layer.

[vinaygouda.ttssl]

@omria @nina-vilela @Omer plz any one can give the solution for this plz ---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
Cell In[5], line 17
13 runner.load_model_script(“”.join(model_script_commands))
16 # Call Optimize to perform the optimization process
—> 17 runner.optimize(calib_dataset)
19 # Save the result state to a Quantized HAR file
20 quantized_model_har_path = f"{model_name}_quantized_model.har"

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_common/states/states.py:16, in allowed_states..wrap..wrapped_func(self, *args, **kwargs)
12 if self._state not in states:
13 raise InvalidStateException(
14 f"The execution of {func.name} is not available under the state: {self._state.value}",
15 )
—> 16 return func(self, *args, **kwargs)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py:2093, in ClientRunner.optimize(self, calib_data, data_type, work_dir)
2061 @allowed_states(States.HAILO_MODEL, States.FP_OPTIMIZED_MODEL)
2062 def optimize(self, calib_data, data_type=CalibrationDataType.auto, work_dir=None):
2063 “”"
2064 Apply optimizations to the model:
2065
(…)
2091
2092 “”"
→ 2093 self._optimize(calib_data, data_type=data_type, work_dir=work_dir)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_common/states/states.py:16, in allowed_states..wrap..wrapped_func(self, *args, **kwargs)
12 if self._state not in states:
13 raise InvalidStateException(
14 f"The execution of {func.name} is not available under the state: {self._state.value}",
15 )
—> 16 return func(self, *args, **kwargs)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py:1935, in ClientRunner._optimize(self, calib_data, data_type, work_dir)
1933 if self._state == States.HAILO_MODEL:
1934 self._optimize_full_precision(calib_data=calib_data, data_type=data_type)
→ 1935 self._sdk_backend.full_quantization(calib_data, data_type=data_type, work_dir=work_dir)
1936 self._state = States.QUANTIZED_MODEL

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1045, in SDKBackendQuantization.full_quantization(self, data, data_type, work_dir, force_results_by_layer)
1043 self.pre_quantization_structural()
1044 params_pre_quantization = copy.deepcopy(self.get_params_pre_quantization())
→ 1045 self._full_acceleras_run(self.calibration_data, data_type)
1046 self._logger.verbose(“Core and post Quantization is done with Acceleras”)
1047 self._finalize_quantization()

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py:1229, in SDKBackendQuantization._full_acceleras_run(self, data, data_type)
1224 else:
1225 # the compression level and the optimization level are not set to the recommended values
1226 self._logger.info(
1227 f"To obtain best performance for models with number of parameters larger than {self.hw_arch.HAILO15_LARGE_MODEL_PARAMS_TH}, it is recommended to use optimization level and compression level {RECOMMENDED_OPTM_LEVEL}."
1228 )
→ 1229 optimization_flow.run()
1230 fp_after_optimization = optimization_flow.get_acceleras_params()
1231 self.save_and_load_params_hailo_optimized(fp_after_optimization)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py:306, in flow_control_method..wrapper(self, *args, **kwargs)
302 setattr(self, attr_name, method_wrapper(method, sup))
304 try:
305 # Execute the original run function
→ 306 return func(self, *args, **kwargs)
308 finally:
309 # Restore original methods after run execution
310 self._call_history = sup.export() if sup.current_checkpoint else None

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py:316, in OptimizationFlow.run(self, memento, run_until)
314 for ind, step_func in enumerate(step_funcs):
315 self._step_index = ind
→ 316 step_func()

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py:250, in flow_control_method..wrapper..method_wrapper..wrapped(*args, **kwargs)
248 if sup.state == SupervisorState.RUNNING:
249 with sup.visit_node(method_name):
→ 250 result = method(*args, **kwargs)
251 if method_name == run_until:
252 sup.state = SupervisorState.DISABLED

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/subprocess_wrapper.py:122, in subprocess_wrapper..decorator..parent_wrapper(self, *args, **kwargs)
120 self.dist_info = dist_info
121 self.build_model()
→ 122 func(self, *args, **kwargs)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py:341, in OptimizationFlow.step2(self)
338 @subprocess_wrapper(DistributionStrategy.DATA_P, {DistributionStrategy.DATA_P})
339 def step2(self):
340 self.build_fp_model()
→ 341 self.post_quantization_optimization()
342 self._update_quantize_data()

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py:250, in flow_control_method..wrapper..method_wrapper..wrapped(*args, **kwargs)
248 if sup.state == SupervisorState.RUNNING:
249 with sup.visit_node(method_name):
→ 250 result = method(*args, **kwargs)
251 if method_name == run_until:
252 sup.state = SupervisorState.DISABLED

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py:396, in OptimizationFlow.post_quantization_optimization(self)
394 self._adaround()
395 self._block_round_training()
→ 396 self._finetune()

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py:250, in flow_control_method..wrapper..method_wrapper..wrapped(*args, **kwargs)
248 if sup.state == SupervisorState.RUNNING:
249 with sup.visit_node(method_name):
→ 250 result = method(*args, **kwargs)
251 if method_name == run_until:
252 sup.state = SupervisorState.DISABLED

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py:655, in OptimizationFlow._finetune(self)
644 model_fp = self.get_fp_model()
645 finetune = QftRunner(
646 self.model,
647 self._parsed_config,
(…)
653 var_freeze_cond=lambda s: “normalization” in s or “avgpool” in s,
654 )
→ 655 finetune.run()
656 self._finalize_algorithm(finetune)
657 if finetune.get_results().status == AlgorithmStatus.SUCCESSFULLY_DONE:

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/optimization_algorithm.py:50, in OptimizationAlgorithm.run(self)
48 def run(self):
49 self.finalize_config()
—> 50 return super().run()

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/algorithm_base.py:151, in AlgorithmBase.run(self, memento)
149 self.restore(memento)
150 start_time = time.time()
→ 151 self._run_int()
152 self._results.status = AlgorithmStatus.SUCCESSFULLY_DONE
153 except KeyboardInterrupt:

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/finetune/qft.py:387, in QftRunner._run_int(self)
386 def _run_int(self):
→ 387 self.run_qft(self._model_native, self._model, metrics=self.metrics)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/finetune/qft.py:507, in QftRunner.run_qft(self, hmodel_fp, hmodel_quant, metrics)
502 tensorboard_writer = TensorboardWriter(
503 distiller=qft_distiller, name=f"{self._name}", workdir=self._work_dir
504 )
505 callbacks.append(tensorboard_writer)
→ 507 self.main_train_summary_per_epoch = qft_distiller.fit(
508 self.train_dataset,
509 verbose=1,
510 epochs=self.ft_cfg.epochs,
511 steps_per_epoch=self.steps_per_epoch,
512 validation_data=self.eval_dataset,
513 validation_steps=self.validation_steps,
514 validation_freq=100, # self.decay_epochs,
515 callbacks=callbacks,
516 )
518 except tf.errors.ResourceExhaustedError as e:
519 self._logger.debug(e)

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/utils/traceback_utils.py:70, in filter_traceback..error_handler(*args, **kwargs)
67 filtered_tb = _process_traceback_frames(e.traceback)
68 # To get the full stack trace, call:
69 # tf.debugging.disable_traceback_filtering()
—> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb

File ~/anaconda3/envs/rpi5/lib/python3.10/site-packages/tensorflow/python/eager/execute.py:52, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
50 try:
51 ctx.ensure_initialized()
—> 52 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
53 inputs, attrs, num_outputs)
54 except core._NotOkStatusException as e:
55 if name is not None:

InternalError: Graph execution error:

Detected at node ‘StatefulPartitionedCall_63’ defined at (most recent call last):
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/runpy.py”, line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/runpy.py”, line 86, in _run_code
exec(code, run_globals)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel_launcher.py”, line 18, in
app.launch_new_instance()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/traitlets/config/application.py”, line 1075, in launch_instance
app.start()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/kernelapp.py”, line 739, in start
self.io_loop.start()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/tornado/platform/asyncio.py”, line 205, in start
self.asyncio_loop.run_forever()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/asyncio/base_events.py”, line 603, in run_forever
self._run_once()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/asyncio/base_events.py”, line 1909, in _run_once
handle._run()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/asyncio/events.py”, line 80, in _run
self._context.run(self._callback, *self._args)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/kernelbase.py”, line 545, in dispatch_queue
await self.process_one()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/kernelbase.py”, line 534, in process_one
await dispatch(*args)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/kernelbase.py”, line 437, in dispatch_shell
await result
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/ipkernel.py”, line 362, in execute_request
await super().execute_request(stream, ident, parent)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/kernelbase.py”, line 778, in execute_request
reply_content = await reply_content
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/ipkernel.py”, line 449, in do_execute
res = shell.run_cell(
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/ipykernel/zmqshell.py”, line 549, in run_cell
return super().run_cell(*args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/interactiveshell.py”, line 3075, in run_cell
result = self._run_cell(
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/interactiveshell.py”, line 3130, in _run_cell
result = runner(coro)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/async_helpers.py”, line 128, in pseudo_sync_runner
coro.send(None)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/interactiveshell.py”, line 3334, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/interactiveshell.py”, line 3517, in run_ast_nodes
if await self.run_code(code, result, async=asy):
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/IPython/core/interactiveshell.py”, line 3577, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File “/tmp/ipykernel_6372/2545589706.py”, line 17, in
runner.optimize(calib_dataset)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_common/states/states.py”, line 16, in wrapped_func
return func(self, *args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py”, line 2093, in optimize
self._optimize(calib_data, data_type=data_type, work_dir=work_dir)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_common/states/states.py”, line 16, in wrapped_func
return func(self, *args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/runner/client_runner.py”, line 1935, in _optimize
self._sdk_backend.full_quantization(calib_data, data_type=data_type, work_dir=work_dir)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py”, line 1045, in full_quantization
self._full_acceleras_run(self.calibration_data, data_type)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_sdk_client/sdk_backend/sdk_backend.py”, line 1229, in _full_acceleras_run
optimization_flow.run()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py”, line 306, in wrapper
return func(self, *args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py”, line 316, in run
step_func()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py”, line 250, in wrapped
result = method(*args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/subprocess_wrapper.py”, line 122, in parent_wrapper
func(self, *args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py”, line 341, in step2
self.post_quantization_optimization()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py”, line 250, in wrapped
result = method(*args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py”, line 396, in post_quantization_optimization
self._finetune()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/tools/orchestator.py”, line 250, in wrapped
result = method(*args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/flows/optimization_flow.py”, line 655, in _finetune
finetune.run()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/optimization_algorithm.py”, line 50, in run
return super().run()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/algorithm_base.py”, line 151, in run
self._run_int()
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/finetune/qft.py”, line 387, in _run_int
self.run_qft(self._model_native, self._model, metrics=self.metrics)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/finetune/qft.py”, line 507, in run_qft
self.main_train_summary_per_epoch = qft_distiller.fit(
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/utils/traceback_utils.py”, line 65, in error_handler
return fn(*args, **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/engine/training.py”, line 1685, in fit
tmp_logs = self.train_function(iterator)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/engine/training.py”, line 1284, in train_function
return step_function(self, iterator)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/engine/training.py”, line 1268, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/engine/training.py”, line 1249, in run_step
outputs = model.train_step(data)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/acceleras/model/distiller.py”, line 133, in train_step
self.optimizer.apply_gradients(zip(factor_gradients, self.trainable_vars))
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/hailo_model_optimization/algorithms/finetune/optimizers.py”, line 103, in apply_gradients
spec[“optimizer”].apply_gradients(spec[“gv”], **kwargs)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/optimizers/optimizer.py”, line 1174, in apply_gradients
return super().apply_gradients(grads_and_vars, name=name)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/optimizers/optimizer.py”, line 650, in apply_gradients
iteration = self._internal_apply_gradients(grads_and_vars)
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/optimizers/optimizer.py”, line 1200, in _internal_apply_gradients
return tf.internal.distribute.interim.maybe_merge_call(
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/optimizers/optimizer.py”, line 1250, in _distributed_apply_gradients_fn
distribution.extended.update(
File “/home/gpu-server/anaconda3/envs/rpi5/lib/python3.10/site-packages/keras/optimizers/optimizer.py”, line 1245, in apply_grad_to_update_var
return self._update_step_xla(grad, var, id(self._var_key(var)))
Node: ‘StatefulPartitionedCall_63’
libdevice not found at ./libdevice.10.bc
[[{{node StatefulPartitionedCall_63}}]] [Op:__inference_train_function_408175]

[vinaygouda.ttssl]

after converting model to hef the detections are not as accepted can you help me out

[vinaygouda.ttssl]

why yolov8 nmspostprocess shows N/A even i pass nms config.json while converstion model from yolov8n to hef model plz give ans @Omer @nina-vilela @omria @Nadav

[nina-vilela]

The kernel and activation are N/A because they are not relevant for the nms postprocess.

[nina-vilela]

@vinaygouda.ttssl could you please describe your issue?

[vinaygouda.ttssl]

@nina-vilela Our pipeline includes processing RTSP streams to detect vehicles, followed by cropping the detected number plates for OCR processing. The issue arises during the resizing step of the cropped number plate images: the resized images do not retain sufficient quality, often appearing blurred or distorted, which prevents accurate OCR text extraction.

We are looking for guidance or suggestions on optimizing this process, particularly with respect to resizing techniques that preserve image clarity for reliable OCR accuracy. Any insights you could provide to enhance our image processing for optimal OCR results would be greatly appreciated.

[vinaygouda.ttssl]

@nina-vilela

@vinaygouda.ttssl could you please describe your issue?

@nina-vilela @omria @Omer i have issues which is currently facing is
Our pipeline includes processing RTSP streams to detect vehicles, followed by cropping the detected number plates for OCR processing. The issue arises during the resizing step of the cropped number plate images: the resized images do not retain sufficient quality, often appearing blurred or distorted, which prevents accurate OCR text extraction.

We are looking for guidance or suggestions on optimizing this process, particularly with respect to resizing techniques that preserve image clarity for reliable OCR accuracy. Any insights you could provide to enhance our image processing for optimal OCR results would be greatly appreciated.

[vinaygouda.ttssl]

@nina-vilela @Omer @omria and anther thing is it possible to extract quelty image or frame from the pipline

[nina-vilela]

@vinaygouda.ttssl

The issue arises during the resizing step of the cropped number plate images: the resized images do not retain sufficient quality, often appearing blurred or distorted, which prevents accurate OCR text extraction.

One possible cause could be performing the cropping after the resize that is done to the model input dimensions.

If the original input has higher dimension, the cropping should be performed from there whenever possible.

[vinaygouda.ttssl]

can you please elaborate it

[nina-vilela]

You should ensure that the license plates are cropped from the original input stream (not resized).

[vinaygouda.ttssl]

@nina-vilela i am using you hailio-rpis-examples in that i modified the app_callback() is as below

This is the callback function that will be called when data is available from the pipeline

def app_callback(pad, info, user_data):

# Get the GstBuffer from the probe info
buffer = info.get_buffer()
# Check if the buffer is valid
if buffer is None:
    return Gst.PadProbeReturn.OK
    
# Using the user_data to count the number of frames
user_data.increment()
string_to_print = f"Frame count: {user_data.get_count()}\n"

# Get the caps from the pad
format, width, height = get_caps_from_pad(pad)

# If the user_data.use_frame is set to True, we can get the video frame from the buffer
frame = None
if format is not None and width is not None and height is not None:
    # Get video frame
    frame = get_numpy_from_buffer(buffer, format, width, height)
print('frame size',width,height)
# Get the detections from the buffer
roi = hailo.get_roi_from_buffer(buffer)
detections = roi.get_objects_typed(hailo.HAILO_DETECTION)
n=len(detections)

boxes = np.zeros((n, 4))
confidence = np.zeros(n)
class_id = np.zeros(n)
#tracker_id = np.empty(n)


for i, detection in enumerate(detections):
    class_id[i] = int(detection.get_class_id())
    confidence[i] = detection.get_confidence()
    #if class_id[i] in [1.0,2.0,3.0,4.0,5.0,10.0,11.0,14.0]:
    #tracker_id[i] = detection.get_objects_typed(hailo.HAILO_UNIQUE_ID)[0].get_id()
    bbox = detection.get_bbox()
    boxes[i] = [int(bbox.xmin() * width), int(bbox.ymin() * height), int(bbox.xmax() * width), int(bbox.ymax() * height)]
    print(f"class name {labels[int(class_id[i])]} conf {confidence[i]} bbox {boxes[i]} ")

hailodetections = sv.Detections(
        xyxy=boxes, 
        confidence=confidence, 
        class_id=class_id,
        )


vehicle_detections = hailodetections[
                (hailodetections.class_id != 6.) & (hailodetections.class_id != 7.) & 
                (hailodetections.class_id != 8.) & (hailodetections.class_id != 9.) & 
                (hailodetections.class_id != 12.) & (hailodetections.class_id != 13.)
            ]
            
vehicle_detections = tracker.update_with_detections(vehicle_detections)
print(vehicle_detections)

numberplatedetection=hailodetections[(hailodetections.class_id == 6.)]
print("number plate ",numberplatedetection)

# Parse the detections
detection_count = 0
orignal_frame_height, original_frame_width, _ = frame.shape
tr1_frame_wh=1000
tr2_frame_wh=250
tr1_np_width,tr1_np_height=150,100

original_frame_height, original_frame_width, _ = frame.shape
tr1_frame_wh = 1000
tr2_frame_w = 400
tr2_frame_h = 400
tr1_np_width, tr1_np_height = 200, 150

for bbox, tr_id, conf in zip(vehicle_detections.xyxy, vehicle_detections.tracker_id, vehicle_detections.confidence):
    if len(numberplatedetection.xyxy) > 0:
        for nbbox, nconf in zip(numberplatedetection.xyxy, numberplatedetection.confidence):
            if containment_checking(bbox, nbbox):
                xmin, ymin, xmax, ymax = map(int, bbox)
                print("v",xmin, ymin, xmax, ymax )
                nxmin, nymin, nxmax, nymax = map(int, nbbox)
                print("before scaling ",nxmin, nymin, nxmax, nymax)
                
                
                # Crop the vehicle image
                cropped_vehicle = frame[ymin:ymax, xmin:xmax]
                cpframe_height, cpframe_width, _ = cropped_vehicle.shape
                
                # Define scaling factors
                scale_x = tr2_frame_w / cpframe_width
                scale_y = tr2_frame_h / cpframe_height
                print(scale_x,scale_y)

                # Adjust number plate coordinates to the resized vehicle frame
                adj_nxmin = int((nxmin - xmin) * scale_x)
                adj_nymin = int((nymin - ymin) * scale_y)
                adj_nxmax = int((nxmax - xmin) * scale_x)
                adj_nymax = int((nymax - ymin) * scale_y)
                
                # Ensure adjusted coordinates are valid
                adj_nxmin = max(0, adj_nxmin)
                adj_nymin = max(0, adj_nymin)
                adj_nxmax = min(tr2_frame_w, adj_nxmax)
                adj_nymax = min(tr2_frame_h, adj_nymax)
                
                print("after scaling ",nxmin, nymin, nxmax, nymax)
                # Resize the vehicle frame for further processing
                
                resized_vehicle_frame = cv2.resize(cv2.cvtColor(cropped_vehicle, cv2.COLOR_BGR2RGB),(tr2_frame_w, tr2_frame_h),cv2.INTER_CUBIC)
                #cv2.rectangle(resized_vehicle_frame,(adj_nxmin,adj_nymin),(adj_nxmax,adj_nymax),(50,250,255),2,cv2.LINE_4)
                
                # Crop the number plate from the resized vehicle frame
                cropped_numberplate = resized_vehicle_frame[adj_nymin:adj_nymax, adj_nxmin:adj_nxmax]
                
                
                processed_image=preprocess_number_plate(cropped_numberplate)
                cv2.imwrite(f'./images/numberplate_{randint(1, 1000)}.jpg', cropped_numberplate)
                """
                extract_numberplate=extract_numberplate_paddle_ocr(cropped_numberplate)
                print(extract_numberplate)
                if extract_numberplate is not None:
                    verified_numberplate=rechck_numberplate(extract_numberplate)
                    
                cv2.imwrite(f'./images/numberplate_{randint(1, 1000)}.jpg', resized_vehicle_frame)
                """
                
                # Save the cropped number plate image
               

                

print(string_to_print)
return Gst.PadProbeReturn.OK