Project 4

import cv2
import numpy as np

class LaneDetection:
    
    frame = np.array([])
    bw_frame = np.array([])
    gaussian_blurred_image = np.array([])
    canny_edge_image = None
    masked_frame= None
    hough_lines = ''
    average_lines = np.array([])
    output = np.array([])
    
    def __init__(self, frame):
        self.frame = frame
    
    def gaussian_smoothening(self, kernel_dims, sigma):
        if len(self.frame.shape) == 3:
            self.bw_frame = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
        else:
            self.bw_frame = self.frame
        self.gaussian_blurred_image = cv2.GaussianBlur(self.bw_frame, kernel_dims, sigma)
#         cv2.imshow("gaussian", self.gaussian_blurred_image)
#         cv2.waitKey(1)
        
    def canny_edge_detector(self, low_lim: int, up_lim: int):
        if self.gaussian_blurred_image.shape != None: 
            result = cv2.Canny(self.gaussian_blurred_image, low_lim, up_lim)
            self.canny_edge_image = np.array(result , dtype=np.float32)
        else:
            pass
        
    def set_frame_mask(self):
        height = self.frame.shape[0]
        width = self.frame.shape[1]
        color = 255
        
        mask = np.zeros((height, width), dtype=np.float32)
        polygon = np.array([
            [(0, height), (round(width*0.3), round(height*0.6)), (round(width*0.55), round(height*0.5)),(round(width*0.9), height)]
        ])
#         polygon = np.array([
#             [(0, height), (round(width * 0.4), round(height*0.5)), (width, height)]
#         ])
        mask = cv2.fillPoly(mask, polygon, color)
        masked_image = cv2.bitwise_and(self.canny_edge_image, mask)

        self.masked_frame = masked_image.astype(np.uint8)
#         cv2.imshow("frame", mask)
#         cv2.waitKey(1)
#         cv2.imshow("masked",masked_image)
#         cv2.waitKey(1)
    
    def determine_hough_lines(self, rho, threshold, minLineLength, maxLineGap):
        self.hough_lines = cv2.HoughLinesP(
            self.masked_frame,
            rho=rho, 
            theta=np.pi/180,
            threshold=threshold,
            lines = np.array([]),
            minLineLength=minLineLength,
            maxLineGap= maxLineGap
                       )
        
    def get_coordinates(self, params):
        slope = params[0]
        intercept = params[1]
        
        y1 = self.frame.shape[0]
        x1 = int((y1-intercept)/slope)
        y2 = int(y1 * 0.75)
        x2= int((y2 - intercept)/slope)
        return np.array([x1, y1, x2, y2])
    
    def avg_lines(self):
        left = []
        right = []
        
        if type(self.hough_lines) is not type(None):
#             print(type(self.hough_lines), self.hough_lines)
            for line in self.hough_lines:
                x1, y1, x2, y2 = line.reshape(4)
                params = np.polyfit((x1, x2), (y1, y2), 1)
                slope = params[0]
                y_intercept = params[1]

                if slope < 0:
                    left.append([slope, y_intercept])
                else:
                    right.append([slope, y_intercept])

            left_line = np.array([0, 0, 0, 0])
            right_line = np.array([0, 0, 0, 0])
            if len(left) > 0:
                left_avg = np.average(left, axis=0)
                left_line = self.get_coordinates(left_avg)
            if len(right)> 0:
                right_avg = np.average(right, axis=0)
                right_line = self.get_coordinates(right_avg)
            self.average_lines = np.array([left_line, right_line])        
    
    def draw_line_on_frame(self, thickness):
        line_image = np.zeros_like(self.frame)
        color =[0, 0, 255]
        
        if type(self.average_lines) is not None:
            for x1, y1, x2, y2  in self.average_lines:
#                 if (x1 and x2 in range(self.frame.shape[1])) and \
#                 (y1 and y2 in range(self.frame.shape[2])):
# #                     print(x1, y1, x2, y2)
#                 print(f"""
#                 x1 = {x1}, {x1.dtype}
#                 y1 = {y1}, {y1.dtype}
#                 x2 = {x2}, {x2.dtype}
#                 y2 = {y2}, {y2.dtype}
#                 """)
                cv2.line(
                    line_image,
                    (x1, y1),
                    (x2,y2),
                    color, 
                    thickness
                )
#         cv2.imshow("Lines", line_image)
#         cv2.waitKey(1)
        combined_image = cv2.addWeighted(self.frame, 0.8, line_image, 1, 1)

        self.output = combined_image

video_capture = cv2.VideoCapture("lane_video_test.mp4")

# frame_height = int(video_capture.get(4))
# frame_width = int(video_capture.get(3))
# print(frame_height, frame_width)
# frame_size = (frame_width, frame_height)
# fps = int(video_capture.get(5))
# print(frame_size) 

output = cv2.VideoWriter('lane_detection_video_output.mp4',
                         cv2.VideoWriter_fourcc(*'XVID'),
                         30,
                         (960, 540),
                         True
                        )
while video_capture.isOpened():
    read_flag, frame = video_capture.read()
    
    if not read_flag:
        print("problem loading Video", read_flag)
        break
    lane_det_frame = LaneDetection(frame)
    lane_det_frame.gaussian_smoothening([5,5], 0)
    lane_det_frame.canny_edge_detector(240, 250)
    lane_det_frame.set_frame_mask()
    lane_det_frame.determine_hough_lines(rho=5, threshold = 30, minLineLength=65, maxLineGap=100)
    lane_det_frame.avg_lines()
    lane_det_frame.draw_line_on_frame(5)
    output.write(lane_det_frame.output)
    video = np.hstack([lane_det_frame.canny_edge_image, lane_det_frame.masked_frame])
    video = cv2.resize(video,None,  fx=0.5, fy=0.5)
    cv2.imshow("frame", lane_det_frame.output)
#     cv2.imshow("frame", video)
    cv2.waitKey(1)

video_capture.release()
cv2.destroyAllWindows()

Lane Detection Process Overview:

1. Converting the image to gray scale and smoothening
2. Applying Canny edge Detector
3. Finding the Hough Lines, from edges.
4. Taking Average of the hough lines based on theta value and categorising as left and right lines
5. Using the line coordinates to superimpose lines on the image.

Code and ALgorithm Walkthrough:

video is opened using cv2.videoCapture(file_src, encoder, fps, output_dims, Color)

encoder value is different for different type of output file for example .mp4, .avi, etc. for mp3 here we have used cv2.VideoWriter_fourcc(*'XVID')

then we read the video frame by frame

• consists of a class called LaneDetection ,that takes in the current frame.
• converts the current frame to greyscale if image is in BGR, and then image smoothening is performed using cv2.GaussianBlur.
• A kernel size of 5x5 is used here to create a larger smoothening effect for the next process
• The gaussian blurred image is then fed to the class function canny_edge_detector(low_lim, upper_lim)
• Passing 240 as lower lim to remove as many irrelevant edges. Accepting only 250, strong white pixels as strong edges for the lanes(done to avoid issues caused by central crack on the road).
• Result is stored inclass obj property canny_edge_image
• Once this is done we set a mask on the canny edge image.
• a mask of size image is created with pixel value = 0
• then we create the polygon by supplying an array of points to the function cv2.fillPoly(src, pts, color), where color is set to 255 (white)
• This mask is then supplied to the function cv2.bitwise_and(img1, img2)
• bitwise and compares the two corresponding pixel locations of the image, and if both are high(1 or 255) returns high else returns low (0).
• this helps us to cut out the unnecesseray area in the frame and brings to focus only the pieces of the frame that have the relevant edges for our application.
• this masked frame is then passed on the the cls function determine_hough_lines(rho, threshold, minLineLength, maxLinegap)
• cv2.HoughLinesP is used to find the probabilistic lines in hough space.
• threshold is the minimum number of lines that need to pass through a point in hough space for the line to be considered.
• rho determines the density of points to consider. larger rho would consider a larger area.
• theta determines angle interval of all the lines plotted in hough space that pass through the edge under consideration.
• minLineLength is used to set the minimum length of a line to be considered in order to be considered in hough space probablisitic.
• maxLineGap is used to set the maximum gap between two lines in order to be considered in hough space probabilistic.
• once Houghlines are found out, based on the orientation value(theta) we want to segregrate the lines as lines that are tilted to the right or to the left and average out the lines to form one line for the right lane and one line for the left lane.
• cv2.houghLinesP returns an array of the x1,y1,x2,y2 points of the lines.
• to achieve this, we call the avg_lines() function.
• This function utilises the lines present in the hough lines array.
• we take each line, and with the x1, y1, x2, y2 co-ordinates, fit the points to a single degree polynomial (y=mx+c). This is acheived using the function cv2.polifit(pt1, pt2, degree_of_polynomial).
• this gives us the slope and y intercept.
• Through this we are able to categorise the lines as left and right, using the slope and storing them in separate arrays.
• We then find the average of the left and right lines in the arrays using np.average(array, axis=0). 0 for column wise average and 1 for row wise. column used here to average slope with slope and intercept with intercepts.
• once the average is found, we passs this to the function get_coordinates([slope, intercept]).
• we take requiredvalues for y and find the corresponding x and return as an array.
• This array is then passed to draw lines.
• np.zeroes_like(self.frame) is used to create a matrix of same dimension as frame(BGR) and name it as line_image.
• On this we take the x1, y1, x2, y2 cooridnates from previous step and plot the lines on line_image supplying a color and thickness value, using the function cv2.lines(line_image, pt1, pt2, color, thickness)
• Once line image is formed, we superimpose on original frame(self.frame) using cv2.addWeighted.
• and this combined image is then writted to the output file using output.write(self.output)
• Once the while loop ends the video_capture.release() function unloads video and cv2.destroyAlWindows() closes all windows opened by opencv during the code run.