How To Use Computer Vision For Automated Sorting

api_key = "YOUR_ROBOFLOW_API_KEY"
from inference import InferencePipeline
from inference.core.interfaces.camera.entities import VideoFrame
import cv2
import supervision as sv
import serial
import time
import sys

# Initialize serial communication with Arduino
arduino = serial.Serial('COM4', 9600, timeout=1)
time.sleep(2)  # Allow Arduino to reset

def send_command_to_arduino(command: str):    """Send a command to Arduino via serial."""    try:        arduino.write(command.encode())  # Send command as bytes        print(f"Sent to Arduino: {command}")    except Exception as e:        print(f"Error sending data to Arduino: {e}")

# Initialize annotators
label_annotator = sv.LabelAnnotator()
box_annotator = sv.BoxAnnotator()

# Initialize a variable to store the last sent command
last_command = None

def my_custom_sink(predictions: dict, video_frame: VideoFrame):    global last_command  # Declare the variable as global to retain its value across calls
    # Extract labels from predictions    labels = [p["class"] for p in predictions.get("predictions", [])]    # Convert predictions to Supervision Detections    detections = sv.Detections.from_inference(predictions)    # Annotate the frame    image = label_annotator.annotate(        scene=video_frame.image.copy(), detections=detections, labels=labels    )    image = box_annotator.annotate(image, detections=detections)    # Display the annotated image    cv2.imshow("Predictions", image)
    # Determine the appropriate command based on the labels    if "apple" in labels:        current_command = "G_APPLE"  # Command for good apples    elif "damaged_apple" in labels:        current_command = "D_APPLE"  # Command for damaged apples    else:        current_command = None  # No command for other cases
    # Send the command only if it has changed    if current_command != last_command:        if current_command is not None:  # Only send if there's a valid command            send_command_to_arduino(current_command)        last_command = current_command  # Update the last command
    if cv2.waitKey(1) & 0xFF == ord('q'):        return

pipeline = InferencePipeline.init(    model_id="apple-sorting-2bfhk/2",    video_reference=0,  # Use the default webcam    on_prediction=my_custom_sink,    api_key=api_key,)
pipeline.start()
pipeline.join()

#include <Servo.h>
int servoPin_1=5;
int servoPin_2=6;

Servo good_apple_servo, damaged_apple_servo;

void setup() {  // put your setup code here, to run once:  Serial.begin(9600);  good_apple_servo.attach(servoPin_1);  damaged_apple_servo.attach(servoPin_2);  good_apple_servo.write(0);  damaged_apple_servo.write(0);
}

void loop() {  // put your main code here, to run repeatedly:if (Serial.available() > 0) {        String command = Serial.readStringUntil('\n'); // Read the command        command.trim(); // Remove extra whitespace
        // Debug: Print the received command        Serial.print("Received command: ");        Serial.println(command);
        if (command == "G_APPLE") {          good_apple_servo.write(90); // Opens sort line for good apple          damaged_apple_servo.write(0); // closes sort line for damaged apple
        } else if (command == "D_APPLE") {          damaged_apple_servo.write(90); // Opens sort line for damaged apple          good_apple_servo.write(0); // closes sort line good apple
        }   }
}