PYNQ-DPU Custom Model

[MichaelVallaro11]

I am trying to use Vitis AI 2.5 to create a custom model using the tf_yolov4_coco_416_416_60.3G_2.5 model found in the model_zoo folder. I train it, quantize, and compile for the KR260 but when I go to load it I get this.

Traceback (most recent call last):
File "/home/ubuntu/vs_humble/conveyor.py", line 13, in
overlay.load_model("deploy.xmodel")
File "/usr/local/share/pynq-venv/lib/python3.10/site-packages/pynq_dpu/dpu.py", line 177, in load_model
assert len(subgraphs) == 1
AssertionError

Any Thoughts?

[lidachang-amd]

Hi @MichaelVallaro11 Can you help to add some codes to print the len of the subgraphs before assert "len(subgraphs)==1"

[MAbdelatti]

Thanks @lidachang-amd. I work with Michael on the project, The length is zero. This gave us a feeling that the NN was not quantized/compiled properly.

[MichaelVallaro11]

I ran this code:

from pynq_dpu import DpuOverlay
import os
import time
import numpy as np
import cv2
import random
import colorsys
from matplotlib.patches import Rectangle
import matplotlib.pyplot as plt

overlay = DpuOverlay("dpu.bit")
overlay.load_model("yolov3.xmodel")

assert len(subgraphs) == 1, "The number of subgraphs is not equal to 1."

anchor_list = [10,13,16,30,33,23,30,61,62,45,59,119,116,90,156,198,373,326]
anchors = np.array([float(x) for x in anchor_list]).reshape(-1, 2)

Get model classification information

def get_class(classes_path):
with open(classes_path) as f:
class_names = f.readlines()
return [c.strip() for c in class_names]

classes_path = "robot.txt"
class_names = get_class(classes_path)
num_classes = len(class_names)
colors = [colorsys.hsv_to_rgb(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
colors = [(int(c[0] * 255), int(c[1] * 255), int(c[2] * 255)) for c in colors]
random.seed(0)
random.shuffle(colors)
random.seed(None)

def letterbox_image(image, size):
ih, iw, _ = image.shape
w, h = size
scale = min(w/iw, h/ih)
nw, nh = int(iwscale), int(ihscale)
new_image = np.ones((h,w,3), dtype=np.uint8) * 128
new_image[(h-nh)//2:(h-nh)//2+nh, (w-nw)//2:(w-nw)//2+nw, :] = cv2.resize(image, (nw,nh))
return new_image

def pre_process(image, model_image_size):
image = image[...,::-1] # BGR to RGB
if model_image_size != (None, None):
assert model_image_size[0] % 32 == 0 and model_image_size[1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(image, tuple(reversed(model_image_size)))
else:
new_image_size = (image.shape[1] - (image.shape[1] % 32), image.shape[0] - (image.shape[0] % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32') / 255.
return np.expand_dims(image_data, 0)

dpu = overlay.runner
inputTensors = dpu.get_input_tensors()
outputTensors = dpu.get_output_tensors()
shapeIn = tuple(inputTensors[0].dims)
output_shapes = [tuple(t.dims) for t in outputTensors]

def _get_feats(feats, anchors, num_classes, input_shape):
num_anchors = len(anchors)
anchors_tensor = np.reshape(np.array(anchors, dtype=np.float32), [1, 1, 1, num_anchors, 2])
grid_size = np.shape(feats)[1:3]
nu = num_classes + 5
predictions = np.reshape(feats, [-1, grid_size[0], grid_size[1], num_anchors, nu])
grid_y = np.tile(np.reshape(np.arange(grid_size[0]), [-1, 1, 1, 1]), [1, grid_size[1], 1, 1])
grid_x = np.tile(np.reshape(np.arange(grid_size[1]), [1, -1, 1, 1]), [grid_size[0], 1, 1, 1])
grid = np.concatenate([grid_x, grid_y], axis = -1)
grid = np.array(grid, dtype=np.float32)

box_xy = (1/(1+np.exp(-predictions[..., :2])) + grid) / np.array(grid_size[::-1], dtype=np.float32)
box_wh = np.exp(predictions[..., 2:4]) * anchors_tensor / np.array(input_shape[::-1], dtype=np.float32)
box_confidence = 1/(1+np.exp(-predictions[..., 4:5]))
box_class_probs = 1/(1+np.exp(-predictions[..., 5:]))
return box_xy, box_wh, box_confidence, box_class_probs

def correct_boxes(box_xy, box_wh, input_shape, image_shape):
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = np.array(input_shape, dtype = np.float32)
image_shape = np.array(image_shape, dtype = np.float32)
new_shape = np.around(image_shape * np.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale

box_mins = box_yx - (box_hw / 2.)
box_maxes = box_yx + (box_hw / 2.)
boxes = np.concatenate([
    box_mins[..., 0:1],
    box_mins[..., 1:2],
    box_maxes[..., 0:1],
    box_maxes[..., 1:2]
], axis = -1)
boxes *= np.concatenate([image_shape, image_shape], axis = -1)
return boxes

def boxes_and_scores(feats, anchors, classes_num, input_shape, image_shape):
box_xy, box_wh, box_confidence, box_class_probs = _get_feats(feats, anchors, classes_num, input_shape)
boxes = correct_boxes(box_xy, box_wh, input_shape, image_shape)
boxes = np.reshape(boxes, [-1, 4])
box_scores = box_confidence * box_class_probs
box_scores = np.reshape(box_scores, [-1, classes_num])
return boxes, box_scores

Define buffers for DPU

input_data = [np.empty(shapeIn, dtype=np.float32, order="C")]
output_data = [np.empty(shape, dtype=np.float32, order="C") for shape in output_shapes]
image = input_data[0]

def draw_bbox(image, bboxes, classes):

num_classes = len(classes)
image_h, image_w, _ = image.shape
hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))

random.seed(0)
random.shuffle(colors)
random.seed(None)

for i, bbox in enumerate(bboxes):
    coor = np.array(bbox[:4], dtype=np.int32)
    fontScale = 0.5
    score = bbox[4]
    class_ind = int(bbox[5])
    bbox_color = colors[class_ind]
    bbox_thick = int(0.6 * (image_h + image_w) / 600)
    c1, c2 = (coor[0], coor[1]), (coor[2], coor[3])
    cv2.rectangle(image, c1, c2, bbox_color, bbox_thick)
return image

def nms_boxes(boxes, scores):

x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]

areas = (x2-x1+1)*(y2-y1+1)
order = scores.argsort()[::-1]

keep = []
while order.size > 0:
    i = order[0]
    keep.append(i)

    xx1 = np.maximum(x1[i], x1[order[1:]])
    yy1 = np.maximum(y1[i], y1[order[1:]])
    xx2 = np.minimum(x2[i], x2[order[1:]])

    w1 = np.maximum(0.0, xx2 - xx1 + 1)
    h1 = np.maximum(0.0, yy2 - yy1 + 1)
    inter = w1 * h1

    ovr = inter / (areas[i] + areas[order[1:]] - inter)
    inds = np.where(ovr <= 0.55)[0]  # threshold
    order = order[inds + 1]

return keep

def evaluate(yolo_outputs, image_shape, class_names, anchors):
score_thresh = 0.2
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
boxes = []
box_scores = []
input_shape = np.shape(yolo_outputs[0])[1 : 3]
input_shape = np.array(input_shape)*32

for i in range(len(yolo_outputs)):
    _boxes, _box_scores = boxes_and_scores(
        yolo_outputs[i], anchors[anchor_mask[i]], len(class_names), 
        input_shape, image_shape)
    boxes.append(_boxes)
    box_scores.append(_box_scores)
boxes = np.concatenate(boxes, axis = 0)
box_scores = np.concatenate(box_scores, axis = 0)

mask = box_scores >= score_thresh
boxes_ = []
scores_ = []
classes_ = []
for c in range(len(class_names)):
    class_boxes_np = boxes[mask[:, c]]
    class_box_scores_np = box_scores[:, c]
    class_box_scores_np = class_box_scores_np[mask[:, c]]
    nms_index_np = nms_boxes(class_boxes_np, class_box_scores_np) 
    class_boxes_np = class_boxes_np[nms_index_np]
    class_box_scores_np = class_box_scores_np[nms_index_np]
    classes_np = np.ones_like(class_box_scores_np, dtype = np.int32) * c
    boxes_.append(class_boxes_np)
    scores_.append(class_box_scores_np)
    classes_.append(classes_np)
boxes_ = np.concatenate(boxes_, axis = 0)
scores_ = np.concatenate(scores_, axis = 0)
classes_ = np.concatenate(classes_, axis = 0)

return boxes_, scores_, classes_

Run object detection

def draw_boxes(image, boxes, scores, classes):
_, ax = plt.subplots(1)
ax.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
image_h, image_w, _ = image.shape

for i, bbox in enumerate(boxes):
    [top, left, bottom, right] = bbox
    width, height = right - left, bottom - top
    center_x, center_y = left + width*0.5, top + height*0.5
    score, class_index = scores[i], classes[i]
    label = '{}: {:.4f}'.format(class_names[class_index], score) 
    color = tuple([color/255 for color in colors[class_index]])
    ax.add_patch(Rectangle((left, top), width, height,
                           edgecolor=color, facecolor='none'))
    ax.annotate(label, (center_x, center_y), color=color, weight='bold', 
                fontsize=12, ha='center', va='center')
return ax

def run(image_path, display=False):
input_image = cv2.imread(image_path)
image_data = pre_process(input_image, (416, 416))
image[0,...] = image_data.reshape(shapeIn[1:])
job_id = dpu.execute_async(input_data, output_data)
dpu.wait(job_id)
yolo_outputs = [np.reshape(data, shape) for data, shape in zip(output_data, output_shapes)]
boxes, scores, classes = evaluate(yolo_outputs, input_image.shape[:2], class_names, anchors)
if display:
_ = draw_boxes(input_image, boxes, scores, classes)
print(f"Number of detected objects: {len(boxes)}")
print(classes)

image_path = 'img/2.jpeg'

run(image_path, display=True)

del overlay, dpu

and here is my output when I added, print(f"Number of subgraphs: {len(subgraphs)}") to dpu.py
Number of subgraphs: 0
Traceback (most recent call last):
File "/home/ubuntu/cv/vs_humble/conveyor.py", line 13, in
overlay.load_model("yolov3.xmodel")
File "/usr/local/share/pynq-venv/lib/python3.10/site-packages/pynq_dpu/dpu.py", line 178, in load_model
assert len(subgraphs) == 1
AssertionError