experimentations/sound2vid_effects.py

142 lines
4.2 KiB
Python

from audio2numpy import open_audio
import cv2
import numpy as np
import scipy.io.wavfile as wf
import scipy.signal as sig
# import matplotlib.pyplot as plt
import thyaudio as ta
def butter_filter(data, cru, btype="low"):
sos1 = sig.butter(10, cru, btype=btype, fs=sample_rate, output="sos")
filt = sig.sosfilt(sos1, data)
return filt
vid_path = "path/to/video_file"
cap = cv2.VideoCapture(vid_path)
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
frame_rate = int(cap.get(cv2.CAP_PROP_FPS))
video_duration = frame_count / frame_rate
print(f"Video Frame Rate: {frame_rate}")
print(f"Video Duration: {video_duration} seconds")
buf = np.empty((frame_count, frame_height, frame_width, 3), np.dtype('uint8'))
print(f"array shape: {buf.shape} frame count: {frame_count}. Width: {frame_width}. Height: {frame_height}")
fc = 0
ret = True
while (fc < frame_count and ret):
ret, buf[fc] = cap.read()
fc += 1
cap.release()
path_to_file = "path/to/audio_file"
if path_to_file.endswith(".mp3"):
sound, sample_rate = open_audio(path_to_file)
elif path_to_file.endswith(".wav"):
sample_rate, sound = wf.read(path_to_file)
## if stereo to mono ##
try:
sound = sound[:, 0] + sound[:, 1]
except IndexError:
pass
sound = sound / np.max(np.abs(sound))
audio_arr_len = np.size(sound)
audio_duration = audio_arr_len / sample_rate
print(f"Initial Audio Duration: {audio_duration} Samples: {audio_arr_len}")
if audio_duration < video_duration:
extra = int((video_duration * sample_rate) - audio_arr_len + (1 / frame_rate))
extra_zeros = np.zeros(extra)
sound = np.concatenate((sound, extra_zeros))
print("ADDED EXTRA ")
print(f"Final Audio Duration: {(extra / sample_rate) + (audio_arr_len / sample_rate)}")
if audio_duration > video_duration:
sound = sound[:int(video_duration * sample_rate)]
print("CHOPPED AUDIO")
print(f"Final Audio Duration: {np.size(sound) / sample_rate}")
## In case it needs filtering. cru=bandwith. btype also "low" or "high"
sound = butter_filter(sound, cru=(200, 1000), btype="bandpass")
print(sound.shape)
print(f"rate {sample_rate}")
samples = np.size(sound)
duration = samples / sample_rate
samples_per_frame = 1
new_samples_per_second = frame_rate * samples_per_frame
resize_samples = int(new_samples_per_second * duration)
print(f"sound duration: {duration} samples: {samples}")
print(f"new_samples_per_second: {new_samples_per_second}")
print(f"resize_samples: {new_samples_per_second * duration} int resize_samples {resize_samples}")
if resize_samples < frame_count:
resize_samples = resize_samples + 1
envelope = ta.ThyEnveloper(sound, window_size=256).get_envelope()
#envelope = np.clip(envelope, 0.0, 0.45)
invert_envelope = ta.ThyEnveloper(sound, window_size=256, invert=True).get_envelope()
resampled_factor = 150
resampled_add = 0
resampled = (sig.resample(envelope, resize_samples) * resampled_factor) + resampled_add
resampled = np.clip(resampled, 0.0, 100)
invert_factor = 150
invert_add = 0
invert_resampled = (sig.resample(invert_envelope, resize_samples) * resampled_factor) \
+ invert_add
print(f"resampled shape: {resampled.shape}")
resampled = np.uint8(resampled)
invert_resampled = np.uint8(invert_resampled)
x = np.linspace(0, 2 * np.pi * frame_count, frame_count)
y0 = (np.sin(x * 20) + 1) * 30
y0 = np.round(y0).astype(int)
y1 = (np.sin(x * 7) + 1) * 40
y1 = np.round(y1).astype(int)
ys = np.round(sound * 150).astype(int)
ys = ys[44000:]
## Vertical frame lines
for i in range(frame_count):
buf[i, :, 1300+y1[i]:, 0] += invert_resampled[i]
ys = ys[2:]
for p in range(frame_height):
buf[i, p, 800+ys[p]:818+ys[p], :] = 0
# buf[i, p, 900+ys[p]:1080, :] = 0
## Horizontal frame lines
# buf[i, 1000+y0[i]:, :, 0] += invert_resampled[i]
for p in range(frame_width):
buf[i, 900+ys[p]:1080, p, :] = 0
# ## Write video from array
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
writer = cv2.VideoWriter("path/to/destination/output_video.mp4", fourcc, frame_rate, (frame_width, frame_height))
print("got this far")
for i in range(frame_count):
image = (buf[i])
writer.write(image)
writer.release()