In the RK3562 Buildroot Linux system environment, we switch between two cameras to capture their data and display it on the LCD screen. Each time we switch between cameras, we stop the capture operation of the original camera, then capture from another camera, and directly write the camera data to the framebuffer for display.We use V4L2 to operate and open two USB cameras, without needing to preview them simultaneously; we only need to switch the preview display between the two cameras. To improve the switching speed, we do not execute the full process from open to close, but instead, after opening, we control the camera capture using stream on and off. However, the following issue arises:After the VIDIOC_STREAMOFF operation, the system gets stuck at the VIDIOC_DQBUF operation after VIDIOC_STREAMON.Upon investigation,VIDIOC_STREAMOFF clears the related memory mapping,so we need to mmap the memory again to function normally. Below is the corresponding mmap operation code:
for(int i = 0; i < 4; i++){ mapbuffer.index = i; ret = ioctl(fd, VIDIOC_QUERYBUF, &mapbuffer); // Query buffer information if(ret < 0) perror("Failed to query buffer queue"); mmpaddr[i] = (unsigned char *)mmap(NULL, mapbuffer.length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, mapbuffer.m.offset); // mapbuffer.m.offset is the offset of the mapped file addr_length[i] = mapbuffer.length; // Add to queue ret = ioctl(fd, VIDIOC_QBUF, &mapbuffer); if(ret < 0) perror("Failed to add to queue");}We first check if there is space in the buffer; if there is, we perform memory mapping and add it to the queue.Below is the complete code:1. Makefile
CC=../buildroot/output/rockchip_rk3562/host/bin/aarch64-linux-gcc
all: $(CC) camera_preview.c -ljpeg -o camera_preview
clean: rm -rf camera_preview
2. camera_preview.c
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <linux/videodev2.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/time.h>
#include <jpeglib.h>
#include <linux/fb.h>
int fd_fb;int screen_size; // Screen pixel size
int LCD_width; // LCD width
int LCD_height; // LCD height
unsigned char *fbbase = NULL; // LCD memory address
unsigned long line_length; // Length of one line of LCD (in bytes)
unsigned int bpp; // Pixel depth
// Initialize LCD
int LCD_Init(void){
struct fb_var_screeninfo var; /* Current var */
struct fb_fix_screeninfo fix; /* Current fix */
fd_fb = open("/dev/fb0", O_RDWR);
if(fd_fb < 0) {
perror("Failed to open LCD");
return -1;
}
// Get LCD information
ioctl(fd_fb, FBIOGET_VSCREENINFO, &var);
// Get variable screen information
ioctl(fd_fb, FBIOGET_FSCREENINFO, &fix);
// LCD_width = var.xres * var.bits_per_pixel / 8; // pixel_width = var.bits_per_pixel / 8;
screen_size = var.xres * var.yres * var.bits_per_pixel / 8;
LCD_width = var.xres;
LCD_height = var.yres;
bpp = var.bits_per_pixel;
line_length = fix.line_length;
printf("LCD resolution: %d %d\n", LCD_width, LCD_height);
printf("bpp: %d\n", bpp);
printf("line length: %ld\n", line_length);
fbbase = mmap(NULL, screen_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_fb, 0); // Map
if (fbbase == (unsigned char *)-1) {
printf("can't mmap\n");
return -1;
}
memset(fbbase, 0xFF, screen_size); // Set LCD to white background
return 0;
}
int LCD_JPEG_Show(const char *JpegData, int size){
int min_hight = LCD_height, min_width = LCD_width, valid_bytes;
struct jpeg_decompress_struct cinfo;
struct jpeg_error_mgr jerr;
cinfo.err = jpeg_std_error(&jerr); // Bind error handling object to decompression object
// Create decompression object
jpeg_create_decompress(&cinfo);
// Specify data source for decompression
jpeg_mem_src(&cinfo, JpegData, size);
// Read image information
jpeg_read_header(&cinfo, TRUE);
// printf("JPEG image size: %d*%d\n", cinfo.image_width, cinfo.image_height);
// Set decoding parameters
cinfo.out_color_space = JCS_RGB; // Can be omitted, defaults to RGB
// cinfo.scale_num = 1; // cinfo.scale_denom = 1; // Set image scaling, scale_num/scale_denom scaling ratio, defaults to 1
// Start decompression
jpeg_start_decompress(&cinfo);
// Allocate space for buffer
unsigned char* jpeg_line_buf = malloc(cinfo.output_components * cinfo.output_width);
unsigned int* fb_line_buf = malloc(line_length); // Each member has 4 bytes corresponding to RGB888
// Determine which resolution is lower, image or LCD
if(cinfo.output_width < min_width) min_width = cinfo.output_width;
if(cinfo.output_height < min_hight) min_hight = cinfo.output_height;
// Read data, data is read line by line
valid_bytes = min_width * bpp / 8; // Effective bytes for one line, actual data size written to LCD memory
unsigned char *ptr = fbbase;
while(cinfo.output_scanline < min_hight) {
jpeg_read_scanlines(&cinfo, &jpeg_line_buf, 1); // Read one line at a time
// Convert BGR888 data read to RGB888
unsigned int red, green, blue;
unsigned int color;
for(int i = 0; i < min_width; i++) {
red = jpeg_line_buf[i*3];
green = jpeg_line_buf[i*3+1];
blue = jpeg_line_buf[i*3+2];
color = red<<16 | green << 8 | blue;
fb_line_buf[i] = color;
}
memcpy(ptr, fb_line_buf, valid_bytes);
ptr += LCD_width*bpp/8;
}
// Finish decompression
jpeg_finish_decompress(&cinfo);
// Destroy decompression object
jpeg_destroy_decompress(&cinfo);
// Free memory
free(jpeg_line_buf);
free(fb_line_buf);
return 1;
}
int main(int argc, char**argv){
if(argc != 3) {
printf("%s </dev/video0,1...> </dev/video0,1...>\n", argv[0]);
return -1;
}
if(LCD_Init() != 0) {
return -1;
}
// 1. Open camera devices
int fd = open(argv[1], O_RDWR);
if(fd < 0) {
perror("Failed to open device");
return -1;
}
int fd2 = open(argv[2], O_RDWR);
if(fd2 < 0) {
perror("Failed to open device 2");
return -1;
}
// 2. Set camera capture format
struct v4l2_format vfmt;
vfmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; // Select video capture
vfmt.fmt.pix.width = 1920; // LCD_width; // Set width, set to LCD width and height
vfmt.fmt.pix.height = 1080; // LCD_height; // Set height
vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; // Set video capture pixel format
int ret = ioctl(fd, VIDIOC_S_FMT, &vfmt); // VIDIOC_S_FMT: Set capture format
if(ret < 0) {
perror("Error setting capture format");
}
memset(&vfmt, 0, sizeof(vfmt));
vfmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd, VIDIOC_G_FMT, &vfmt);
if(ret < 0) {
perror("Failed to read capture format");
}
printf("Set resolution width = %d\n", vfmt.fmt.pix.width);
printf("Set resolution height = %d\n", vfmt.fmt.pix.height);
unsigned char *p = (unsigned char*)&vfmt.fmt.pix.pixelformat;
printf("pixelformat = %c%c%c%c\n", p[0],p[1],p[2],p[3]);
struct v4l2_format vfmt2;
vfmt2.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; // Select video capture
vfmt2.fmt.pix.width = 1920; // LCD_width; // Set width, set to LCD width and height
vfmt2.fmt.pix.height = 1080; // LCD_height; // Set height
vfmt2.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; // Set video capture pixel format
ret = ioctl(fd2, VIDIOC_S_FMT, &vfmt2); // VIDIOC_S_FMT: Set capture format
if(ret < 0) {
perror("Error setting capture format 2");
}
memset(&vfmt2, 0, sizeof(vfmt2));
vfmt2.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd2, VIDIOC_G_FMT, &vfmt2);
if(ret < 0) {
perror("Failed to read capture format 2");
}
printf("2 Set resolution width = %d\n", vfmt2.fmt.pix.width);
printf("2 Set resolution height = %d\n", vfmt2.fmt.pix.height);
unsigned char *p2 = (unsigned char*)&vfmt2.fmt.pix.pixelformat;
printf("2 pixelformat = %c%c%c%c\n", p2[0],p2[1],p2[2],p2[3]);
// 4. Request buffer queue
struct v4l2_requestbuffers reqbuffer;
reqbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
reqbuffer.count = 4; // Request 4 buffers
reqbuffer.memory = V4L2_MEMORY_MMAP; // Use memory-mapped method
ret = ioctl(fd, VIDIOC_REQBUFS, &reqbuffer);
if(ret < 0) {
perror("Failed to request buffer queue");
}
struct v4l2_requestbuffers reqbuffer2;
reqbuffer2.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
reqbuffer2.count = 4; // Request 4 buffers
reqbuffer2.memory = V4L2_MEMORY_MMAP; // Use memory-mapped method
ret = ioctl(fd2, VIDIOC_REQBUFS, &reqbuffer2);
if(ret < 0) {
perror("Failed to request buffer queue 2");
}
// Map, need to query buffer information before mapping -> map each buffer one by one -> add buffer to queue
struct v4l2_buffer mapbuffer;
unsigned char *mmpaddr[4]; // Used to store mapped addresses
unsigned int addr_length[4]; // Store size of mapped space
mapbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; // Initialize type
/*for(int i = 0; i < 4; i++) {
mapbuffer.index = i;
ret = ioctl(fd, VIDIOC_QUERYBUF, &mapbuffer); // Query buffer information
if(ret < 0)
perror("Failed to query buffer queue");
mmpaddr[i] = (unsigned char *)mmap(NULL, mapbuffer.length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, mapbuffer.m.offset); // mapbuffer.m.offset is the offset of the mapped file
addr_length[i] = mapbuffer.length; // Add to queue
ret = ioctl(fd, VIDIOC_QBUF, &mapbuffer);
if(ret < 0)
perror("Failed to add to queue");
}*/
struct v4l2_buffer mapbuffer2;
unsigned char *mmpaddr2[4]; // Used to store mapped addresses
unsigned int addr_length2[4]; // Store size of mapped space
mapbuffer2.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; // Initialize type
/*for(int i = 0; i < 4; i++) {
mapbuffer2.index = i;
ret = ioctl(fd2, VIDIOC_QUERYBUF, &mapbuffer2); // Query buffer information
if(ret < 0)
perror("Failed to query buffer queue 2");
mmpaddr2[i] = (unsigned char *)mmap(NULL, mapbuffer2.length, PROT_READ|PROT_WRITE, MAP_SHARED, fd2, mapbuffer2.m.offset); // mapbuffer.m.offset is the offset of the mapped file
addr_length2[i] = mapbuffer2.length; // Add to queue
ret = ioctl(fd2, VIDIOC_QBUF, &mapbuffer2);
if(ret < 0)
perror("Failed to add to queue 2");
}*/
#if 0
// Open device
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd, VIDIOC_STREAMON, &type);
if(ret < 0) perror("Failed to open device");
// unsigned char rgbdata[LCD_height*LCD_width*3]; // Store decoded RGB data
while(1) {
// Extract a frame of data from the queue
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd, VIDIOC_DQBUF, &readbuffer); // Get a frame of data from the buffer queue (dequeue)
// After dequeuing, get the index of the cached index, get the corresponding cached mapped address mmpaddr[readbuffer.index]
if(ret < 0) perror("Failed to get data");
// Display on LCD
LCD_JPEG_Show(mmpaddr[readbuffer.index], readbuffer.length);
// After reading data, add the buffer back to the queue
ret = ioctl(fd, VIDIOC_QBUF, &readbuffer);
if(ret < 0) perror("Failed to add to queue");
}
// Close device
ret = ioctl(fd, VIDIOC_STREAMOFF, &type);
if(ret < 0) perror("Failed to close device");
#endif
struct timeval tv;
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
struct v4l2_buffer readbuffer, readbuffer2;
int j = 0;
while(1) {
/*ret = ioctl(fd, VIDIOC_REQBUFS, &reqbuffer);
if(ret < 0) {
perror("Failed to request buffer queue");
}*/
for(int i = 0; i < 4; i++) {
mapbuffer.index = i;
ret = ioctl(fd, VIDIOC_QUERYBUF, &mapbuffer); // Query buffer information
if(ret < 0) perror("Failed to query buffer queue");
mmpaddr[i] = (unsigned char *)mmap(NULL, mapbuffer.length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, mapbuffer.m.offset); // mapbuffer.m.offset is the offset of the mapped file
addr_length[i] = mapbuffer.length; // Add to queue
ret = ioctl(fd, VIDIOC_QBUF, &mapbuffer);
if(ret < 0) perror("Failed to add to queue");
}
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd, VIDIOC_STREAMON, &type);
if(ret < 0) perror("Failed to open device");
gettimeofday(&tv, NULL);
printf("1 Current time in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
j = 0;
while (j++ < 60) {
ret = ioctl(fd, VIDIOC_DQBUF, &readbuffer);
if(ret < 0) perror("Failed to get data");
if (j == 1) {
gettimeofday(&tv, NULL);
printf("1 get first frame in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
}
LCD_JPEG_Show(mmpaddr[readbuffer.index], readbuffer.length);
if (j == 1) {
gettimeofday(&tv, NULL);
printf("1 show first frame in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
}
ret = ioctl(fd, VIDIOC_QBUF, &readbuffer);
if(ret < 0) perror("Failed to add to queue");
}
gettimeofday(&tv, NULL);
printf("2 Current time in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
ret = ioctl(fd, VIDIOC_STREAMOFF, &type);
if(ret < 0) perror("Failed to close device");
/*ret = ioctl(fd2, VIDIOC_REQBUFS, &reqbuffer2);
if(ret < 0) {
perror("Failed to request buffer queue 2");
}*/
for(int i = 0; i < 4; i++) {
mapbuffer2.index = i;
ret = ioctl(fd2, VIDIOC_QUERYBUF, &mapbuffer2); // Query buffer information
if(ret < 0) perror("Failed to query buffer queue 2");
mmpaddr2[i] = (unsigned char *)mmap(NULL, mapbuffer2.length, PROT_READ|PROT_WRITE, MAP_SHARED, fd2, mapbuffer2.m.offset); // mapbuffer.m.offset is the offset of the mapped file
addr_length2[i] = mapbuffer2.length; // Add to queue
ret = ioctl(fd2, VIDIOC_QBUF, &mapbuffer2);
if(ret < 0) perror("Failed to add to queue 2");
}
readbuffer2.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd2, VIDIOC_STREAMON, &type);
if(ret < 0) perror("Failed to open device 2");
gettimeofday(&tv, NULL);
printf("3 Current time in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
j = 0;
while (j++ < 60) {
ret = ioctl(fd2, VIDIOC_DQBUF, &readbuffer2);
if(ret < 0) perror("Failed to get data 2");
if (j == 1) {
gettimeofday(&tv, NULL);
printf("2 get first frame in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
}
LCD_JPEG_Show(mmpaddr2[readbuffer2.index], readbuffer2.length);
if (j == 1) {
gettimeofday(&tv, NULL);
printf("2 show first frame in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
}
ret = ioctl(fd2, VIDIOC_QBUF, &readbuffer2);
if(ret < 0) perror("Failed to add to queue 2");
}
gettimeofday(&tv, NULL);
printf("4 Current time in milliseconds since epoch: %ld\n", (tv.tv_sec * 1000) + (tv.tv_usec / 1000));
ret = ioctl(fd2, VIDIOC_STREAMOFF, &type);
if(ret < 0) perror("Failed to close device 2");
}
// Unmap
for(int i = 0; i < 4; i++) {
munmap(mmpaddr[i], addr_length[i]);
munmap(mmpaddr2[i], addr_length2[i]);
}
close(fd);
close(fd2);
return 0;
}From the complete code, we can also see the modifications made during our debugging process, where we commented out the relevant process content that caused runtime errors.