Computer Vision

Skills

1. Python

2. ffmpeg,

3. glob

4. json

5. open cv

6. sklearn

Happening frame selection - k-means clustering to cluster frames on hisogram

import cv2

from google.colab.patches import cv2_imshow

import sys

import os

from PIL import Image

#import Image

import matplotlib.image as mpimg

import matplotlib.pyplot as plt

import numpy as np

import math

from sklearn.decomposition import PCA

def histogram_vector(image,n):

hist0 = cv2.calcHist(image, [0], None, [n], [0, n])

hist1 = cv2.calcHist(image, [1], None, [n], [0, n])

hist2 = cv2.calcHist(image, [2], None, [n], [0, n])

hist3 = cv2.calcHist(image, [3], None, [n], [0, n])

return np.array([hist1,hist2,hist3]).reshape(3*n)

#hist_vector = histogram_vector(image,8)

# print(hist_vector.T.shape)

# print(image.T[0].shape)

def pca_image(image,components):

l1 = PCA(n_components=components)

l1.fit(image.T[0])

l2 = PCA(n_components=components)

l2.fit(image.T[1])

l3 = PCA(n_components=components)

l3.fit(image.T[2])

l = [l1.singular_values_, l2.singular_values_, l3.singular_values_ ]

return np.array(l).reshape(3*components)

# l = pca_image(image,8)

# print(l.shape)

# data = np.array((l,l,l,l,l))

# print(data.shape)

# data = data.T

# print(data.shape)

# data = list(zip(data))

# print(np.array(data))

fileName = '/content/drive/MyDrive/Project Docments/Soccernet/Video summarization/18.mkv'

videoCap = cv2.VideoCapture(fileName)

data = np.empty((24,))

number_of_frames = int(videoCap.get(cv2.CAP_PROP_FRAME_COUNT))

for i in range(number_of_frames-1):

success, image = videoCap.read()

l = histogram_vector(image,8)

data = np.vstack((data,l))

data = data[1:number_of_frames]

height = len(image)

width = len(image[0])

cv2.destroyAllWindows()

videoCap.release()

success, image = videoCap.read()

print(width)

print(height)

print(np.array(image).shape)

print(data.shape)

print(number_of_frames)

import matplotlib.pyplot as plt

from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=16)

kmeans.fit(data)

lbl = kmeans.labels_

print(lbl)

plt.scatter(data.T[0], data.T[1], c=kmeans.labels_)

plt.show()

fourcc = cv2.VideoWriter_fourcc(*'XVID')

outputVideo = cv2.VideoWriter('output.avi', fourcc, 20.0, (width,height))

videoCap = cv2.VideoCapture(fileName)

lbl = list(lbl)

unique_frame_index = np.sort( np.unique(lbl, return_index=True)[1] )

for i in range(number_of_frames):

success, image = videoCap.read()

if i in set(unique_frame_index):

print(lbl[i])

#cv2_imshow(image)

outputVideo.write(image)

cv2.destroyAllWindows()

outputVideo.release()

import cv2

from google.colab.patches import cv2_imshow

import sys

import os

from PIL import Image

#import Image

import matplotlib.image as mpimg

import numpy as np

import math

from sklearn.decomposition import PCA

import matplotlib.pyplot as plt

from sklearn.cluster import KMeans

def pca_image(image,components):

l1 = PCA(n_components=components)

l1.fit(image.T[0])

l2 = PCA(n_components=components)

l2.fit(image.T[1])

l3 = PCA(n_components=components)

l3.fit(image.T[2])

l = [l1.singular_values_, l2.singular_values_, l3.singular_values_ ]

return np.array(l).reshape(3*components)

# l = pca_image(image,8)

# print(l.shape)

def histogram_vector(image,n):

hist0 = cv2.calcHist(image, [0], None, [n], [0, n])

hist1 = cv2.calcHist(image, [1], None, [n], [0, n])

hist2 = cv2.calcHist(image, [2], None, [n], [0, n])

hist3 = cv2.calcHist(image, [3], None, [n], [0, n])

return np.array([hist1,hist2,hist3]).reshape(3*n)

#hist_vector = histogram_vector(image,8)

# print(hist_vector.T.shape)

# print(image.T[0].shape)

# fileName = '/content/drive/MyDrive/Project Docments/lip sync/vid/3.mp4'

fileName = input("Enter video path - ")

videoCap = cv2.VideoCapture(fileName)

#number_of_frames = int(input("number_of_frames - "))

number_of_frames = int(videoCap.get(cv2.CAP_PROP_FRAME_COUNT))

expected_number_of_frames = int(input("expected_number_of_frames - "))

hidden_features = 255

data = np.empty((hidden_features*3,))

for i in range(number_of_frames-1):

success, image = videoCap.read()

l = histogram_vector(image,hidden_features)

data = np.vstack((data,l))

#print(i)

data = data[1:number_of_frames]

height = len(image)

width = len(image[0])

cv2.destroyAllWindows()

videoCap.release()

kmeans = KMeans(n_clusters=expected_number_of_frames)

kmeans.fit(data)

lbl = kmeans.labels_

# print(lbl)

# plt.scatter(data.T[0], data.T[1], c=kmeans.labels_)

# plt.show()

fourcc = cv2.VideoWriter_fourcc(*'XVID')

outputVideo = cv2.VideoWriter('output.avi', fourcc, 20.0, (width,height))

videoCap = cv2.VideoCapture(fileName)

lbl = list(lbl)

unique_frame_index = np.sort( np.unique(lbl, return_index=True)[1] )

for i in range(number_of_frames):

success, image = videoCap.read()

if i in set(unique_frame_index):

#print(lbl[i])

#cv2_imshow(image)

outputVideo.write(image)

cv2.destroyAllWindows()

outputVideo.release()

No class action video trim - randomly generate a time in millisecond. Find if an action class falls into the 8 second span around the randomly generated time

import json

import random

import numpy as np

import os

from tkinter import PROJECTING

import ffmpeg

import glob

#when json is given function will output possible no class time stamps

def readJSON(json_file_path):

f = open(json_file_path)

data = json.load(f)

return data

def generate_random_time(MinuteStart,MinuteEnd):

minute = random.randint(MinuteStart,MinuteEnd)

second = random.randint(4,55)

return minute,second

def generate_random_milisecond(millisecondstart,millisecondstop): #one game in 0 to 2,700,000 (45 min) long

millisecond = random.randint(millisecondstart,millisecondstop)

return millisecond

def No_Class_or_Not(position,millisecond,fourSecInMillseconds): #out puts whether it is a no class of not (boolean)

condition = np.logical_and(millisecond-fourSecInMillseconds < np.array(position), np.array(position) < millisecond+fourSecInMillseconds )

return condition.any()

def trim(in_file, out_file, start, end):

if os.path.exists(out_file):

os.remove(out_file)

in_file_probe_result = ffmpeg.probe(in_file)

in_file_duration = in_file_probe_result.get(

"format", {}).get("duration", None)

#print(in_file_duration)

input_stream = ffmpeg.input(in_file)

pts = "PTS-STARTPTS"

video = input_stream.trim(start=start, end=end).setpts(pts)

audio = (input_stream

.filter_("atrim", start=start, end=end)

.filter_("asetpts", pts))

video_and_audio = ffmpeg.concat(video, audio, v=1, a=1)

output = ffmpeg.output(video_and_audio, out_file, format="mp4")

output.run()

out_file_probe_result = ffmpeg.probe(out_file)

out_file_duration = out_file_probe_result.get(

"format", {}).get("duration", None)

print(out_file_duration)

fourSecInMillseconds = 4000

k = 0

output_folder = '/content/drive/MyDrive/SoccerNet/NoClass/'

for leagues in glob.iglob('/content/drive/MyDrive/SoccerNet/test/*'):

for years in glob.iglob(leagues+'/*'):

for games in glob.iglob(years + '/*'):

content = []

for files in glob.iglob(games + '/*'):

content.append(files)

video_1_path = content[0]

video_2_path = content[1]

JSON_path = content[2]

# print(video_2_path)

t = readJSON(JSON_path)

GameOnePosition = []

GameTwoPosition = []

for i in range( len(t["annotations"]) ):

if(int(t["annotations"][i]['gameTime'][0]) ==1 ):

GameOnePosition.append(int(t["annotations"][i]['position']))

elif (int(t["annotations"][i]['gameTime'][0]) ==2 ):

GameTwoPosition.append(int(t["annotations"][i]['position']))

for i in range(10):

millisecond = generate_random_milisecond(0 , 2700000)

if ( not(No_Class_or_Not(GameOnePosition,millisecond,fourSecInMillseconds)) ):

k = k+1

output_path = f'{output_folder}{k}.mkv'

#print(output_path)

start_time = ( millisecond- fourSecInMillseconds/2 )/1000

end_time = ( millisecond+ fourSecInMillseconds/2 )/1000

trim(video_1_path, output_path, start_time, end_time)

elif ( not(No_Class_or_Not(GameTwoPosition,millisecond,fourSecInMillseconds)) ):

k = k+1

output_path = f'{output_folder}{k}.mkv'

#print(output_path)

start_time = ( millisecond- fourSecInMillseconds/2 )/1000

end_time = ( millisecond+ fourSecInMillseconds/2 )/1000

trim(video_2_path, output_path, start_time, end_time)