Using OpenCV for Image Tracking and Face Recognition on Raspberry Pi

import cv2

a=cv2.imread("/home/pi/2020-06-15-162551_1920x1080_scrot.png")
cv2.imshow("test",a)
cv2.waitKey()
cv2.destroyAllWindows()

import cv2

cap = cv2.VideoCapture(0)
while True:
    ret, frame = cap.read()
    cv2.imshow('frame', frame)
    # This step is necessary, otherwise the image cannot be displayed
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything is done, release the capture
cap.release()
cv2.destroyAllWindows()

import sys
import cv2
import math
import time
import threading
import numpy as np
import HiwonderSDK.yaml_handle as yaml_handle

if sys.version_info.major == 2:
    print('Please run this program with python3!')
    sys.exit(0)

range_rgb = {
    'red':   (0, 0, 255),
    'blue':  (255, 0, 0),
    'green': (0, 255, 0),
    'black': (0, 0, 0),
    'white': (255, 255, 255)}

__target_color = ('red', 'green', 'blue')
lab_data = yaml_handle.get_yaml_data(yaml_handle.lab_file_path)
    
# Find the largest contour
# The parameter is a list of contours to compare

def getAreaMaxContour(contours):
    contour_area_temp = 0
    contour_area_max = 0
    area_max_contour = None

    for c in contours:  # Iterate through all contours
        contour_area_temp = math.fabs(cv2.contourArea(c))  # Calculate contour area
        if contour_area_temp > contour_area_max:
            contour_area_max = contour_area_temp
            if contour_area_temp > 300:  # Only when the area is larger than 300, the largest area contour is valid, to filter interference
                area_max_contour = c

    return area_max_contour, contour_area_max  # Return the largest contour

detect_color = None
color_list = []
start_pick_up = False
size = (640, 480)

def run(img):
    global rect
    global detect_color
    global start_pick_up
    global color_list
        
    img_copy = img.copy()
    frame_resize = cv2.resize(img_copy, size, interpolation=cv2.INTER_NEAREST)
    frame_gb = cv2.GaussianBlur(frame_resize, (3, 3), 3)
    frame_lab = cv2.cvtColor(frame_gb, cv2.COLOR_BGR2LAB)  # Convert image to LAB space
    color_area_max = None
    max_area = 0
    areaMaxContour_max = 0
    if not start_pick_up:
        for i in lab_data:
            if i in __target_color:
                frame_mask = cv2.inRange(frame_lab,
                                             (lab_data[i]['min'][0],
                                              lab_data[i]['min'][1],
                                              lab_data[i]['min'][2]),
                                             (lab_data[i]['max'][0],
                                              lab_data[i]['max'][1],
                                              lab_data[i]['max'][2]))  # Perform bitwise operation on the original image and mask
                opened = cv2.morphologyEx(frame_mask, cv2.MORPH_OPEN, np.ones((3, 3), np.uint8))  # Open operation
                closed = cv2.morphologyEx(opened, cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8))  # Close operation
                contours = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]  # Find contours
                areaMaxContour, area_max = getAreaMaxContour(contours)  # Find the largest contour
                
                if areaMaxContour is not None:
                    if area_max > max_area:  # Find the largest area
                        max_area = area_max
                        color_area_max = i
                        areaMaxContour_max = areaMaxContour
        
        if max_area > 500:  # If the largest area is found
            rect = cv2.minAreaRect(areaMaxContour_max)
            box = np.int0(cv2.boxPoints(rect))
            y = int((box[1][0]-box[0][0])/2+box[0][0])
            x = int((box[2][1]-box[0][1])/2+box[0][1])
            print('X:',x,'Y:',y) # Print coordinates
            cv2.drawContours(img, [box], -1, range_rgb[color_area_max], 2)
            if not start_pick_up:
                if color_area_max == 'red':  # Red is the largest
                    color = 1
                elif color_area_max == 'green':  # Green is the largest
                    color = 2
                elif color_area_max == 'blue':  # Blue is the largest
                    color = 3
                else:
                    color = 0
                color_list.append(color)
                if len(color_list) == 3:  # Multiple judgments
                    # Take the average value
                    color = int(round(np.mean(np.array(color_list))))
                    color_list = []
                    if color == 1:
                        detect_color = 'red'

                    elif color == 2:
                        detect_color = 'green'

                    elif color == 3:
                        detect_color = 'blue'

                    else:
                        detect_color = 'None'

##    cv2.putText(img, "Color: " + detect_color, (10, img.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, detect_color, 2)
    return img

if __name__ == '__main__':
    
    cap = cv2.VideoCapture(-1) # Read camera
    __target_color = ('red',)
    while True:
        ret, img = cap.read()
        if ret:
            frame = img.copy()
            Frame = run(frame)           
            cv2.imshow('Frame', Frame)
            key = cv2.waitKey(1)
            if key == 27:
                break
        else:
            time.sleep(0.01)
    cv2.destroyAllWindows()

import os
import sys
import cv2
import math
import time
import numpy as np
import HiwonderSDK.Misc as Misc

if sys.version_info.major == 2:
    print('Please run this program with python3!')
    sys.exit(0)

__finger = 0
__t1 = 0
__step = 0
__count = 0
__get_finger = False

# Initial position
def initMove():
    pass

def reset():
    global __finger, __t1, __step, __count, __get_finger
    __finger = 0
    __t1 = 0
    __step = 0
    __count = 0
    __get_finger = False
    
def init():
    reset()
    initMove()

class Point(object):  # A coordinate point
    x = 0
    y = 0

    def __init__(self, x=0, y=0):
        self.x = x
        self.y = y

class Line(object):  # A line
    def __init__(self, p1, p2):
        self.p1 = p1
        self.p2 = p2

def GetCrossAngle(l1, l2):
    '''
    Calculate the angle between two lines
    :param l1:
    :param l2:
    :return:
    '''
    arr_0 = np.array([(l1.p2.x - l1.p1.x), (l1.p2.y - l1.p1.y)])
    arr_1 = np.array([(l2.p2.x - l2.p1.x), (l2.p2.y - l2.p1.y)])
    cos_value = (float(arr_0.dot(arr_1)) / (np.sqrt(arr_0.dot(arr_0))
                                            * np.sqrt(arr_1.dot(arr_1))))   # Note to convert to float for calculation
    return np.arccos(cos_value) * (180/np.pi)

def distance(start, end):
    """
    Calculate the distance between two points
    :param start: Starting point
    :param end: Ending point
    :return: Return the distance between the two points
    """
    s_x, s_y = start
    e_x, e_y = end
    x = s_x - e_x
    y = s_y - e_y
    return math.sqrt((x**2)+(y**2))

def image_process(image, rw, rh):  # hsv
    '''
    # Light influence, please modify the range of cb
    # The Cr component of normal yellow-skinned people is about 140~160
    Recognize skin color
    :param image: Image
    :return: Recognized binary image
    '''
    frame_resize = cv2.resize(image, (rw, rh), interpolation=cv2.INTER_CUBIC)
    YUV = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2YCR_CB)  # Convert the image to YCrCb
    _, Cr, _ = cv2.split(YUV)  # Split YCrCb
    Cr = cv2.GaussianBlur(Cr, (5, 5), 0)
    
    _, Cr = cv2.threshold(Cr, 135, 160, cv2.THRESH_BINARY +
                          cv2.THRESH_OTSU)  # OTSU Binarization

    # Open operation, remove noise
    open_element = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
    opend = cv2.morphologyEx(Cr, cv2.MORPH_OPEN, open_element)
    # Erosion
    kernel = np.ones((3, 3), np.uint8)
    erosion = cv2.erode(opend, kernel, iterations=3)

    return erosion

def get_defects_far(defects, contours, img):
    '''
    Get the farthest point in the convex hull
    '''
    if defects is None and contours is None:
        return None
    far_list = []
    for i in range(defects.shape[0]):
        s, e, f, d = defects[i, 0]
        start = tuple(contours[s][0])
        end = tuple(contours[e][0])
        far = tuple(contours[f][0])
        # Calculate the distance between two points
        a = distance(start, end)
        b = distance(start, far)
        c = distance(end, far)
        # Calculate the angle between fingers
        angle = math.acos((b ** 2 + c ** 2 - a ** 2) /
                          (2 * b * c)) * 180 / math.pi
        # The angle between fingers generally does not exceed 100 degrees
        # Less than 90 degrees
        if angle <= 75:  # 90:
            #cv.circle(img, far, 10, [0, 0, 255], 1)
            far_list.append(far)
    return far_list

def get_max_coutour(cou, max_area):
    '''
    Find the largest contour
    Calculate based on area, after finding the maximum, determine whether it is less than the minimum area, and if it is, give up
    :param cou: Contours
    :return: Return the largest contour
    '''
    max_coutours = 0
    r_c = None
    if len(cou) < 1:
        return None
    else:
        for c in cou:
            # Calculate area
            temp_coutours = math.fabs(cv2.contourArea(c))
            if temp_coutours > max_coutours:
                max_coutours = temp_coutours
                cc = c
        # Determine the largest area among all contours
        if max_coutours > max_area:
            r_c = cc
        return r_c

def find_contours(binary, max_area):
    '''
    CV_RETR_EXTERNAL - Only extract the outermost contour
    CV_RETR_LIST - Extract all contours and place them in a list
    CV_RETR_CCOMP - Extract all contours and organize them into a two-layer hierarchy: the top layer is the outer boundary of the connected domain, and the second layer is the inner boundary of the hole.
    CV_RETR_TREE - Extract all contours and reconstruct the entire hierarchy of nested contours
    method  Approximation method (for all nodes, excluding those using internal approximation CV_RETR_RUNS).
    CV_CHAIN_CODE - Output contour of Freeman chain code. Other methods output polygons (vertex sequence).
    CV_CHAIN_APPROX_NONE - Translate all points from chain code form (convert) to point sequence form.
    CV_CHAIN_APPROX_SIMPLE - Compress horizontal, vertical and diagonal segments, that is, the function only retains the end pixel points;
    CV_CHAIN_APPROX_TC89_L1,
    CV_CHAIN_APPROX_TC89_KCOS - Apply the Teh-Chin chain approximation algorithm. CV_LINK_RUNS - Use a completely different contour extraction algorithm by connecting horizontal fragments with a value of 1.
    :param binary: Incoming binary image
    :return: Return the largest contour
    '''
    # Find all contours
    contours = cv2.findContours(
        binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]
    # Return the largest contour
    return get_max_coutour(contours, max_area)

def get_hand_number(binary_image, contours, rw, rh, rgb_image):
    '''
    :param binary_image:
    :param rgb_image:
    :return:
    '''
    # # 2、Find the position of the fingertips
    # # Find contours, return the largest contour
    x = 0
    y = 0
    coord_list = []
    new_hand_list = []  # Get the final finger coordinates

    if contours is not None:
        # Perimeter  0.035 Modify according to recognition situation, the better the recognition, the smaller
        epsilon = 0.020 * cv2.arcLength(contours, True)
        # Contour similarity
        approx = cv2.approxPolyDP(contours, epsilon, True)
        # cv2.approxPolyDP() The parameter 2 (epsilon) is a distance value, which indicates the degree of approach of the polygon contour to the actual contour. The smaller the value, the more accurate; parameter 3 indicates whether it is closed.
        # cv2.polylines(rgb_image, [approx], True, (0, 255, 0), 1)# Draw polygon

        if approx.shape[0] >= 3:  # More than three points are needed for a polygon; a triangle requires three points
            approx_list = []
            for j in range(approx.shape[0]):  # Store all points of the polygon in a list
                # cv2.circle(rgb_image, (approx[j][0][0],approx[j][0][1]), 5, [255, 0, 0], -1)
                approx_list.append(approx[j][0])
            approx_list.append(approx[0][0])    # Add the first point at the end
            approx_list.append(approx[1][0])    # Add the second point at the end

            for i in range(1, len(approx_list) - 1):
                p1 = Point(approx_list[i - 1][0],
                           approx_list[i - 1][1])    # Declare a point
                p2 = Point(approx_list[i][0], approx_list[i][1])
                p3 = Point(approx_list[i + 1][0], approx_list[i + 1][1])
                line1 = Line(p1, p2)    # Declare a line
                line2 = Line(p2, p3)
                angle = GetCrossAngle(line1, line2)     # Get the angle between the two lines
                angle = 180 - angle     #
                # print angle
                if angle < 42:  # Get the angle between the two lines, and if it is less than 37 degrees
                    #cv2.circle(rgb_image, tuple(approx_list[i]), 5, [255, 0, 0], -1)
                    coord_list.append(tuple(approx_list[i]))

        ##############################################################################
        # Remove points between fingers
        # 1、Get the farthest point of the convex hull, the farthest point
        #cv2.drawContours(rgb_image, contours, -1, (255, 0, 0), 1)
        try:
            hull = cv2.convexHull(contours, returnPoints=False)
            # Find convex hull defects. The returned data is 【starting point, ending point, farthest point, approximate distance to the farthest point】
            defects = cv2.convexityDefects(contours, hull)
            # Return the points between fingers
            hand_coord = get_defects_far(defects, contours, rgb_image)
        except:
            return rgb_image, 0
        
        # 2、Remove the farthest point from coord_list
        alike_flag = False
        if len(coord_list) > 0:
            for l in range(len(coord_list)):
                for k in range(len(hand_coord)):
                    if (-10 <= coord_list[l][0] - hand_coord[k][0] <= 10 and
                            -10 <= coord_list[l][1] - hand_coord[k][1] <= 10):    # Compare X and Y axes, remove the close ones
                        alike_flag = True
                        break   #
                if alike_flag is False:
                    new_hand_list.append(coord_list[l])
                alike_flag = False
            # Get the fingertip coordinate list and display
            for i in new_hand_list:
                j = list(tuple(i))
                j[0] = int(Misc.map(j[0], 0, rw, 0, 640))
                j[1] = int(Misc.map(j[1], 0, rh, 0, 480))
                cv2.circle(rgb_image, (j[0], j[1]), 20, [0, 255, 255], -1)
    fingers = len(new_hand_list)

    return rgb_image, fingers


def run(img, debug=False):

    global __act_map, __get_finger
    global __step, __count, __finger

    binary = image_process(img, 320, 240)
    contours = find_contours(binary, 3000)
    img, finger = get_hand_number(binary, contours, 320, 240, img)
    if not __get_finger:
        if finger == __finger:
            __count += 1
        else:
            __count = 0
        __finger = finger
        
    cv2.putText(img, "Finger(s):%d" % __finger, (50, 480 - 30),
                cv2.FONT_HERSHEY_SIMPLEX, 1.2, (0, 255, 255), 2) # Write the number of recognized fingers on the image
    
    return img 

if __name__ == '__main__':
    
    init()
    cap = cv2.VideoCapture(-1) # Read camera
    while True:
        ret, img = cap.read()
        if ret:
            frame = img.copy()
            Frame = run(frame)           
            frame_resize = cv2.resize(Frame, (320, 240))
            cv2.imshow('frame', frame_resize)
            key = cv2.waitKey(1)
            if key == 27:
                break
        else:
            time.sleep(0.01)
    cv2.destroyAllWindows()

import sys
import cv2
import math
import time
import threading
import numpy as np
import HiwonderSDK.tm1640 as tm
import RPi.GPIO as GPIO

GPIO.setwarnings(False)
GPIO.setmode(GPIO.BCM)

color_range = {
'red': [(0, 101, 177), (255, 255, 255)], 
'green': [(47, 0, 135), (255, 119, 255)], 
'blue': [(0, 0, 0), (255, 255, 115)], 
'black': [(0, 0, 0), (41, 255, 136)], 
'white': [(193, 0, 0), (255, 250, 255)], 
}

if sys.version_info.major == 2:
    print('Please run this program with python3!')
    sys.exit(0)
    
range_rgb = {
    'red': (0, 0, 255),
    'blue': (255, 0, 0),
    'green': (0, 255, 0),
    'black': (0, 0, 0),
    'white': (255, 255, 255),
}

# Find the largest contour
# The parameter is a list of contours to compare

def getAreaMaxContour(contours):
    contour_area_temp = 0
    contour_area_max = 0
    area_max_contour = None

    for c in contours:  # Iterate through all contours
        contour_area_temp = math.fabs(cv2.contourArea(c))  # Calculate contour area
        if contour_area_temp > contour_area_max:
            contour_area_max = contour_area_temp
            if contour_area_temp > 50:  # Only when the area is larger than 50, the largest area contour is valid, to filter interference
                area_max_contour = c

    return area_max_contour, contour_area_max  # Return the largest contour

shape_length = 0

def move():
    global shape_length
    
    while True:
        if shape_length == 3:
            print('Triangle')
            ## Display 'Triangle'
            tm.display_buf = (0x80, 0xc0, 0xa0, 0x90, 0x88, 0x84, 0x82, 0x81,
                              0x81, 0x82, 0x84,0x88, 0x90, 0xa0, 0xc0, 0x80)
            tm.update_display()
            
        elif shape_length == 4:
            print('Rectangle')
            ## Display 'Rectangle'
            tm.display_buf = (0x00, 0x00, 0x00, 0x00, 0xff, 0x81, 0x81, 0x81,
                              0x81, 0x81, 0x81,0xff, 0x00, 0x00, 0x00, 0x00)
            tm.update_display()
            
        elif shape_length >= 6:           
            print('Circle')
            ## Display 'Circle'
            tm.display_buf = (0x00, 0x00, 0x00, 0x00, 0x1c, 0x22, 0x41, 0x41,
                              0x41, 0x22, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00)
            tm.update_display()
            
        time.sleep(0.01)
               
# Run a child thread
th = threading.Thread(target=move)
th.setDaemon(True)
th.start()

shape_list = []
action_finish = True

if __name__ == '__main__':
    
    cap = cv2.VideoCapture(-1)
    while True:
        ret,img = cap.read()
        if ret:
            img_copy = img.copy()
            img_h, img_w = img.shape[:2]
            frame_gb = cv2.GaussianBlur(img_copy, (3, 3), 3)      
            frame_lab = cv2.cvtColor(frame_gb, cv2.COLOR_BGR2LAB)  # Convert image to LAB space
            max_area = 0
            color_area_max = None    
            areaMaxContour_max = 0

            if action_finish:
                for i in color_range:
                    if i != 'white':
                        frame_mask = cv2.inRange(frame_lab, color_range[i][0], color_range[i][1])  # Perform bitwise operation on the original image and mask
                        opened = cv2.morphologyEx(frame_mask, cv2.MORPH_OPEN, np.ones((6,6),np.uint8))  # Open operation
                        closed = cv2.morphologyEx(opened, cv2.MORPH_CLOSE, np.ones((6,6),np.uint8)) # Close operation
                        contours = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]  # Find contours
                        areaMaxContour, area_max = getAreaMaxContour(contours)  # Find the largest contour
                        if areaMaxContour is not None:
                            if area_max > max_area:# Find the largest area
                                max_area = area_max
                                color_area_max = i
                                areaMaxContour_max = areaMaxContour
            if max_area > 200:                   
                cv2.drawContours(img, areaMaxContour_max, -1, (0, 0, 255), 2)
                # Recognize shape
                # Perimeter  0.035 Modify according to recognition situation, the better the recognition, the smaller
                epsilon = 0.035 * cv2.arcLength(areaMaxContour_max, True)
                # Contour similarity
                approx = cv2.approxPolyDP(areaMaxContour_max, epsilon, True)
                shape_list.append(len(approx))
                if len(shape_list) == 30:
                    shape_length = int(round(np.mean(shape_list)))                            
                    shape_list = []
                    print(shape_length)
                    
            frame_resize = cv2.resize(img, (320, 240))
            cv2.imshow('frame', frame_resize)
            key = cv2.waitKey(1)
            if key == 27:
                break
        else:
            time.sleep(0.01)
    my_camera.camera_close()
    cv2.destroyAllWindows()

import cv2
import sys
from pyzbar import pyzbar

if sys.version_info.major == 2:
    print('Please run this program with python3!')
    sys.exit(0)

def run(image):
    # Find barcodes in the image and decode each barcode
    barcodes = pyzbar.decode(image)
    # Loop through detected barcodes
    for barcode in barcodes:
        # Extract the bounding box position of the barcode
        (x, y, w, h) = barcode.rect
        # Draw a box around the barcode in the image
        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)

        barcodeData = barcode.data.decode("utf-8")
        barcodeType = barcode.type
        # Draw the barcode data and barcode type on the image
        text = "{} ({})".format(barcodeData, barcodeType)
        cv2.putText(image, text, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)

    return image

if __name__ == '__main__':
    
    cap = cv2.VideoCapture(-1) # Read camera
    
    while True:
        ret, img = cap.read()
        if ret:
            frame = img.copy()
            Frame = run(frame)           
            cv2.imshow('Frame', Frame)
            key = cv2.waitKey(1)
            if key == 27:
                break
        else:
            time.sleep(0.01)
    cv2.destroyAllWindows()

cd /home/pi/Downloads
sudo pip3 install apriltag-0.0.16-cp37-cp37m-linux_armv7l.whl

import sys
import cv2
import math
import time
import threading
import numpy as np
import apriltag

# apriltag detection

if sys.version_info.major == 2:
    print('Please run this program with python3!')
    sys.exit(0)

object_center_x = 0.0
object_center_y = 0.0

# Detect apriltag
detector = apriltag.Detector(searchpath=apriltag._get_demo_searchpath())
def apriltagDetect(img):
    global object_center_x, object_center_y
    
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    detections = detector.detect(gray, return_image=False)

    if len(detections) != 0:
        for detection in detections:                       
            corners = np.rint(detection.corners)  # Get four corner points
            cv2.drawContours(img, [np.array(corners, np.int)], -1, (0, 255, 255), 2)

            tag_family = str(detection.tag_family, encoding='utf-8')  # Get tag_family
            tag_id = int(detection.tag_id)  # Get tag_id

            object_center_x, object_center_y = int(detection.center[0]), int(detection.center[1])  # Center point
            
            return tag_family, tag_id
            
    return None, None

def run(img):
    global state
    global tag_id
    global action_finish
    global object_center_x, object_center_y
     
    img_h, img_w = img.shape[:2]
    
    tag_family, tag_id = apriltagDetect(img) # apriltag detection
    
    if tag_id is not None:
        print('X:',object_center_x,'Y:',object_center_y)
        cv2.putText(img, "tag_id: " + str(tag_id), (10, img.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.65, [0, 255, 255], 2)
        cv2.putText(img, "tag_family: " + tag_family, (10, img.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, [0, 255, 255], 2)
    else:
        cv2.putText(img, "tag_id: None", (10, img.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.65, [0, 255, 255], 2)
        cv2.putText(img, "tag_family: None", (10, img.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, [0, 255, 255], 2)
    
    return img

if __name__ == '__main__':
    
    cap = cv2.VideoCapture(-1) # Read camera
    
    while True:
        ret, img = cap.read()
        if ret:
            frame = img.copy()
            Frame = run(frame)           
            cv2.imshow('Frame', Frame)
            key = cv2.waitKey(1)
            if key == 27:
                break
        else:
            time.sleep(0.01)
    cv2.destroyAllWindows()

Finally, with pip3 install -U numpy succeeded. Therefore, it is best to use pip3 with python3. There are many attempts on the internet, some have upgraded versions, some have downgraded versions, all kinds of strange phenomena emerge, and the statements vary. Reference:

1121:error:(-2:Unspecified error) FAILED: fs.is_open(). Can’t open

After searching for a long time, I found that there was an extra point at the beginning.