Add some documentation on using models in Frigate #102

csanders commented 2025年02月01日 01:23:48 +01:00

No description provided.

Curid commented 2025年02月12日 21:20:17 +01:00

Since we're moving away from mobiledet models it's important that the Frigate documentation includes instructions for how to patch Frigate so it can use our new yolo models.

frigate 0.14.1

diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py
index 8a54363e..b7c6343a 100644
--- a/frigate/detectors/plugins/cpu_tfl.py
+++ b/frigate/detectors/plugins/cpu_tfl.py
@@ -38,9 +38,58 @@ class CpuTfl(DetectionApi):
 self.tensor_output_details = self.interpreter.get_output_details()
 
 def detect_raw(self, tensor_input):
+ scale, zero_point = self.tensor_input_details[0]["quantization"]
+ tensor_input = (
+ (tensor_input - scale * zero_point * 255) * (1.0 / (scale * 255))
+ ).astype(self.tensor_input_details[0]["dtype"])
+
 self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
 self.interpreter.invoke()
 
+ scale, zero_point = self.tensor_output_details[0]["quantization"]
+ tensor_output = self.interpreter.get_tensor(
+ self.tensor_output_details[0]["index"]
+ )
+ tensor_output = (tensor_output.astype(np.float32) - zero_point) * scale
+ model_input_shape = self.tensor_input_details[0]["shape"]
+
+ tensor_output[:, [0, 2]] *= model_input_shape[2]
+ tensor_output[:, [1, 3]] *= model_input_shape[1]
+ model_box_count = tensor_output.shape[2]
+ probs = tensor_output[0, 4:, :]
+ all_ids = np.argmax(probs, axis=0)
+ all_confidences = probs.T[np.arange(model_box_count), all_ids]
+ all_boxes = tensor_output[0, 0:4, :].T
+ mask = all_confidences > 0.5
+ class_ids = all_ids[mask]
+ confidences = all_confidences[mask]
+ cx, cy, w, h = all_boxes[mask].T
+
+ if model_input_shape[3] == 3:
+ scale_y, scale_x = 1 / model_input_shape[1], 1 / model_input_shape[2]
+ else:
+ scale_y, scale_x = 1 / model_input_shape[2], 1 / model_input_shape[3]
+ detections = np.stack(
+ (
+ class_ids,
+ confidences,
+ scale_y * (cy - h / 2),
+ scale_x * (cx - w / 2),
+ scale_y * (cy + h / 2),
+ scale_x * (cx + w / 2),
+ ),
+ axis=1,
+ )
+ if detections.shape[0] > 20:
+ boxes = np.stack((cx - w / 2, cy - h / 2, w, h), axis=1)
+ indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.5)
+ detections = detections[indexes]
+ if detections.shape[0] > 20:
+ detections = detections[np.argpartition(detections[:, 1], -20)[-20:]]
+ detections = detections.copy()
+ detections.resize((20, 6))
+ return detections
+
 boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0]
 class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0]
 scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0]
diff --git a/frigate/detectors/plugins/edgetpu_tfl.py b/frigate/detectors/plugins/edgetpu_tfl.py
index c320bd89..ee3678e2 100644
--- a/frigate/detectors/plugins/edgetpu_tfl.py
+++ b/frigate/detectors/plugins/edgetpu_tfl.py
@@ -57,9 +57,58 @@ class EdgeTpuTfl(DetectionApi):
 self.model_type = detector_config.model.model_type
 
 def detect_raw(self, tensor_input):
+ scale, zero_point = self.tensor_input_details[0]["quantization"]
+ tensor_input = (
+ (tensor_input - scale * zero_point * 255) * (1.0 / (scale * 255))
+ ).astype(self.tensor_input_details[0]["dtype"])
+
 self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
 self.interpreter.invoke()
 
+ scale, zero_point = self.tensor_output_details[0]["quantization"]
+ tensor_output = self.interpreter.get_tensor(
+ self.tensor_output_details[0]["index"]
+ )
+ tensor_output = (tensor_output.astype(np.float32) - zero_point) * scale
+ model_input_shape = self.tensor_input_details[0]["shape"]
+
+ tensor_output[:, [0, 2]] *= model_input_shape[2]
+ tensor_output[:, [1, 3]] *= model_input_shape[1]
+ model_box_count = tensor_output.shape[2]
+ probs = tensor_output[0, 4:, :]
+ all_ids = np.argmax(probs, axis=0)
+ all_confidences = probs.T[np.arange(model_box_count), all_ids]
+ all_boxes = tensor_output[0, 0:4, :].T
+ mask = all_confidences > 0.5
+ class_ids = all_ids[mask]
+ confidences = all_confidences[mask]
+ cx, cy, w, h = all_boxes[mask].T
+
+ if model_input_shape[3] == 3:
+ scale_y, scale_x = 1 / model_input_shape[1], 1 / model_input_shape[2]
+ else:
+ scale_y, scale_x = 1 / model_input_shape[2], 1 / model_input_shape[3]
+ detections = np.stack(
+ (
+ class_ids,
+ confidences,
+ scale_y * (cy - h / 2),
+ scale_x * (cx - w / 2),
+ scale_y * (cy + h / 2),
+ scale_x * (cx + w / 2),
+ ),
+ axis=1,
+ )
+ if detections.shape[0] > 20:
+ boxes = np.stack((cx - w / 2, cy - h / 2, w, h), axis=1)
+ indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.5)
+ detections = detections[indexes]
+ if detections.shape[0] > 20:
+ detections = detections[np.argpartition(detections[:, 1], -20)[-20:]]
+ detections = detections.copy()
+ detections.resize((20, 6))
+ return detections
+
 boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0]
 class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0]
 scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0]

Since we're moving away from mobiledet models it's important that the Frigate documentation includes instructions for how to patch Frigate so it can use our new yolo models. frigate 0.14.1 ``` diff diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index 8a54363e..b7c6343a 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -38,9 +38,58 @@ class CpuTfl(DetectionApi): self.tensor_output_details = self.interpreter.get_output_details() def detect_raw(self, tensor_input): + scale, zero_point = self.tensor_input_details[0]["quantization"] + tensor_input = ( + (tensor_input - scale * zero_point * 255) * (1.0 / (scale * 255)) + ).astype(self.tensor_input_details[0]["dtype"]) + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) self.interpreter.invoke() + scale, zero_point = self.tensor_output_details[0]["quantization"] + tensor_output = self.interpreter.get_tensor( + self.tensor_output_details[0]["index"] + ) + tensor_output = (tensor_output.astype(np.float32) - zero_point) * scale + model_input_shape = self.tensor_input_details[0]["shape"] + + tensor_output[:, [0, 2]] *= model_input_shape[2] + tensor_output[:, [1, 3]] *= model_input_shape[1] + model_box_count = tensor_output.shape[2] + probs = tensor_output[0, 4:, :] + all_ids = np.argmax(probs, axis=0) + all_confidences = probs.T[np.arange(model_box_count), all_ids] + all_boxes = tensor_output[0, 0:4, :].T + mask = all_confidences > 0.5 + class_ids = all_ids[mask] + confidences = all_confidences[mask] + cx, cy, w, h = all_boxes[mask].T + + if model_input_shape[3] == 3: + scale_y, scale_x = 1 / model_input_shape[1], 1 / model_input_shape[2] + else: + scale_y, scale_x = 1 / model_input_shape[2], 1 / model_input_shape[3] + detections = np.stack( + ( + class_ids, + confidences, + scale_y * (cy - h / 2), + scale_x * (cx - w / 2), + scale_y * (cy + h / 2), + scale_x * (cx + w / 2), + ), + axis=1, + ) + if detections.shape[0] > 20: + boxes = np.stack((cx - w / 2, cy - h / 2, w, h), axis=1) + indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.5) + detections = detections[indexes] + if detections.shape[0] > 20: + detections = detections[np.argpartition(detections[:, 1], -20)[-20:]] + detections = detections.copy() + detections.resize((20, 6)) + return detections + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] diff --git a/frigate/detectors/plugins/edgetpu_tfl.py b/frigate/detectors/plugins/edgetpu_tfl.py index c320bd89..ee3678e2 100644 --- a/frigate/detectors/plugins/edgetpu_tfl.py +++ b/frigate/detectors/plugins/edgetpu_tfl.py @@ -57,9 +57,58 @@ class EdgeTpuTfl(DetectionApi): self.model_type = detector_config.model.model_type def detect_raw(self, tensor_input): + scale, zero_point = self.tensor_input_details[0]["quantization"] + tensor_input = ( + (tensor_input - scale * zero_point * 255) * (1.0 / (scale * 255)) + ).astype(self.tensor_input_details[0]["dtype"]) + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) self.interpreter.invoke() + scale, zero_point = self.tensor_output_details[0]["quantization"] + tensor_output = self.interpreter.get_tensor( + self.tensor_output_details[0]["index"] + ) + tensor_output = (tensor_output.astype(np.float32) - zero_point) * scale + model_input_shape = self.tensor_input_details[0]["shape"] + + tensor_output[:, [0, 2]] *= model_input_shape[2] + tensor_output[:, [1, 3]] *= model_input_shape[1] + model_box_count = tensor_output.shape[2] + probs = tensor_output[0, 4:, :] + all_ids = np.argmax(probs, axis=0) + all_confidences = probs.T[np.arange(model_box_count), all_ids] + all_boxes = tensor_output[0, 0:4, :].T + mask = all_confidences > 0.5 + class_ids = all_ids[mask] + confidences = all_confidences[mask] + cx, cy, w, h = all_boxes[mask].T + + if model_input_shape[3] == 3: + scale_y, scale_x = 1 / model_input_shape[1], 1 / model_input_shape[2] + else: + scale_y, scale_x = 1 / model_input_shape[2], 1 / model_input_shape[3] + detections = np.stack( + ( + class_ids, + confidences, + scale_y * (cy - h / 2), + scale_x * (cx - w / 2), + scale_y * (cy + h / 2), + scale_x * (cx + w / 2), + ), + axis=1, + ) + if detections.shape[0] > 20: + boxes = np.stack((cx - w / 2, cy - h / 2, w, h), axis=1) + indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.5) + detections = detections[indexes] + if detections.shape[0] > 20: + detections = detections[np.argpartition(detections[:, 1], -20)[-20:]] + detections = detections.copy() + detections.resize((20, 6)) + return detections + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] ```

Curid commented 2025年06月30日 08:48:50 +02:00

@harryqt would you like to review the instructions before we merge them? rendered

@harryqt would you like to review the instructions before we merge them? [rendered](https://codeberg.org/Curid/TF-CCTV/src/commit/8407a3a114084751146397eeba02e9b99212c861/docs/frigate.md)

harryqt commented 2025年06月30日 08:52:01 +02:00

Everything is good, except import cv2 statement is missing.

Everything is good, except `import cv2` statement is missing.

csanders commented 2025年07月01日 00:52:20 +02:00

It was missing in the original but it's in the diff: 8407a3a114/docs/frigate-yolo-support-v0.15.1.diff

Lines 8 and 78, unless you're talking about somewhere else?

It was missing in the original but it's in the diff: https://codeberg.org/Curid/TF-CCTV/src/commit/8407a3a114084751146397eeba02e9b99212c861/docs/frigate-yolo-support-v0.15.1.diff Lines 8 and 78, unless you're talking about somewhere else?

harryqt commented 2025年07月01日 00:57:30 +02:00

All good 👍

All good 👍