From: Antonio Ospite Date: Mon, 28 Sep 2015 21:21:46 +0000 (+0200) Subject: Initial import X-Git-Url: https://git.ao2.it/experiments/opencv_trail_effect.git/commitdiff_plain/d095022ca7b1d9eb46521073b71f2327fb9d5637?ds=inline Initial import --- d095022ca7b1d9eb46521073b71f2327fb9d5637 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75be08c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +opencv_trail_effect diff --git a/Frame.hpp b/Frame.hpp new file mode 100644 index 0000000..662954b --- /dev/null +++ b/Frame.hpp @@ -0,0 +1,78 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef FRAME_HPP +#define FRAME_HPP + +#include + +class Frame { +private: + cv::Mat data; + cv::Mat mask; + +public: + Frame(cv::Size size) + { + data = cv::Mat::zeros(size, CV_8UC3); + mask = cv::Mat::zeros(size, CV_8UC1); + } + + Frame(const cv::Mat& frame_data) + { + frame_data.copyTo(data); + } + + Frame(const cv::Mat& frame_data, const cv::Mat& frame_mask) + { + frame_data.copyTo(data); + frame_mask.copyTo(mask); + } + + ~Frame() + { + } + + void copyTo(cv::Mat& destination) + { + cv::Mat converted_data; + + data.convertTo(converted_data, destination.type()); + converted_data.copyTo(destination, mask); + } + + void zero() + { + data.setTo(cv::Scalar(0)); + mask.setTo(cv::Scalar(0)); + } + + cv::Mat getMasked() + { + cv::Mat masked_frame; + + data.copyTo(masked_frame, mask); + return masked_frame; + } + + cv::Mat getData() { return data; } + cv::Mat getMask() { return mask; } +}; + +#endif // FRAME_HPP diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..de166de --- /dev/null +++ b/Makefile @@ -0,0 +1,56 @@ +CXXFLAGS = -ansi -pedantic -pedantic-errors -Wall -g3 -O2 -D_ANSI_SOURCE_ +CXXFLAGS += -fno-common \ + -fvisibility=hidden \ + -Wall \ + -Wextra \ + -Wformat=2 \ + -Winit-self \ + -Winline \ + -Wpacked \ + -Wpointer-arith \ + -Wlarger-than-65500 \ + -Wmissing-declarations \ + -Wmissing-format-attribute \ + -Wmissing-noreturn \ + -Wredundant-decls \ + -Wsign-compare \ + -Wstrict-aliasing=2 \ + -Wswitch-enum \ + -Wundef \ + -Wunreachable-code \ + -Wunsafe-loop-optimizations \ + -Wunused-but-set-variable \ + -Wwrite-strings \ + -Wp,-D_FORTIFY_SOURCE=2 \ + -fstack-protector \ + --param=ssp-buffer-size=4 + +CXXFLAGS += $(shell pkg-config --cflags opencv) +LDLIBS += $(shell pkg-config --libs opencv) + +opencv_trail_effect: $(wildcard *.hpp) + + +# Some command lines to imitate different trail styles + +blame_it_on_the_boogie: opencv_trail_effect + ./opencv_trail_effect -l 12 -s background -d fadeaccumulate + +wtf: opencv_trail_effect + ./opencv_trail_effect -l -1 -s background -d copy + +l_anima_vola: opencv_trail_effect + ./opencv_trail_effect -l 30 -s background -d copy -r + +average: opencv_trail_effect + ./opencv_trail_effect -l 10 -s background -d average -B + +clean: + rm -f opencv_trail_effect vgdump gtk.suppression + +test: opencv_trail_effect + [ -f gtk.suppression ] || wget -nv https://people.gnome.org/~johan/gtk.suppression + G_DEBUG=gc-friendly G_SLICE=always-malloc \ + valgrind --tool=memcheck --leak-check=full --leak-resolution=high \ + --num-callers=20 --log-file=vgdump \ + --suppressions=gtk.suppression ./opencv_trail_effect diff --git a/README b/README new file mode 100644 index 0000000..cd242b0 --- /dev/null +++ b/README @@ -0,0 +1,63 @@ +opencv_trail_effect is an experiment about recreating the "trail effect"[1], +which can be seen in some cool music videos, automatically and in real-time, +and even without the use of a green-screen. + +[1] http://dailypsychedelicvideo.com/tag/trail-effect/ + +Some examples of it are: + + The Jacksons - Blame It On The Boogie - https://www.youtube.com/watch?v=mkBS4zUjJZo + OK Go - WTF? - https://www.youtube.com/watch?v=12zJw9varYE + Elisa - L'Anima Vola - https://www.youtube.com/watch?v=MqhxIQD16EA + +opencv_trail_effect draws a trail after performing foreground segmentation +(using thresholding or background subtraction) using OpenCV. + +The "sole" assumptions for a decent effect are that: + + 1. the camera does not move; + 2. the background lighting in the scene is quite stable (when doing + background subtraction it is recommended to disable auto-gain in the + camera, and avoiding any artifact introduced by the power line + frequency). + + +Examples of use +--------------- + +The effect in "Blame It On The Boogie" could be described as a short faded +trail and can be achieved with this command line: + + $ ./opencv_trail_effect -l 12 -s background -d fadeaccumulate + + +The effect in "WTF?" is equivalent to an infinite trail in which the isolated +foreground is copied on the background and stays there, it can be achieved +with this command line: + + $ ./opencv_trail_effect -l -1 -s background -d copy + + +The effect in "L'Anima Vola" seems to have a "catch-up" behavior; that one can +be done by drawing the trail in reverse, with this command line: + + $ ./opencv_trail_effect -l 30 -s background -d copy -r + + +Another effect seen in movies can be obtained by doing the average of the +frames in the trail, possibly with the most recent frame drawn on top: + + $ ./opencv_trail_effect -l 10 -s background -d average -B + + +Side note +------------------- + +These effects can bring to mind other cool effects in music videos, like the +frame shuffling effect in Peter Gabriel's Sledgehammer +(https://www.youtube.com/watch?v=OJWJE0x7T4Q), that one can be done with the +nervousTV plugin from effectv +(https://fukuchi.org/research/effectv/index.html.en) or frei0r filters +(https://www.dyne.org/software/frei0r/), for example with: + + $ gst-launch-1.0 -v v4l2src ! videoconvert ! frei0r-filter-nervous ! videoconvert ! autovideosink diff --git a/Segmentation.hpp b/Segmentation.hpp new file mode 100644 index 0000000..bd9b4c9 --- /dev/null +++ b/Segmentation.hpp @@ -0,0 +1,94 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef SEGMENTATION_HPP +#define SEGMENTATION_HPP + +#include + +class Segmentation { +public: + virtual cv::Mat getForegroundMask(const cv::Mat& frame) = 0; + virtual ~Segmentation() {} +}; + +class DummySegmentation : public Segmentation { +public: + cv::Mat getForegroundMask(const cv::Mat& frame) + { + return cv::Mat(frame.size(), CV_8UC1, cv::Scalar(1)); + } +}; + +class ThresholdSegmentation : public Segmentation { +private: + int threshold; + +public: + ThresholdSegmentation(int threshold_value) + { + threshold = threshold_value; + } + + cv::Mat getForegroundMask(const cv::Mat& frame) + { + cv::Mat gray_frame; + cv::Mat frame_mask; + + cvtColor(frame, gray_frame, CV_RGB2GRAY); + cv::threshold(gray_frame, frame_mask, threshold, 255, CV_THRESH_TOZERO); + + return frame_mask; + } +}; + +/* + * Some background on... background subtraction can be found here: + * http://docs.opencv.org/master/d1/dc5/tutorial_background_subtraction.html + */ + +class MOG2Segmentation : public Segmentation, public cv::BackgroundSubtractorMOG2 { +public: + MOG2Segmentation(cv::VideoCapture& inputVideo, int learning_frames) : + cv::BackgroundSubtractorMOG2() + { + cv::Mat background; + cv::Mat foreground_mask; + + for (int i = 0; i < learning_frames; i++) { + inputVideo >> background; + this->operator()(background, foreground_mask); + } + } + + cv::Mat getForegroundMask(const cv::Mat& frame) + { + cv::Mat foreground_mask; + + this->operator()(frame, foreground_mask, 0); + cv::erode(foreground_mask, foreground_mask, cv::Mat()); + cv::dilate(foreground_mask, foreground_mask, cv::Mat()); + cv::threshold(foreground_mask, foreground_mask, 0, 255, CV_THRESH_OTSU); + cv::medianBlur(foreground_mask, foreground_mask, 9); + + return foreground_mask; + } +}; + +#endif // SEGMENTATION_HPP diff --git a/TODO b/TODO new file mode 100644 index 0000000..8aa8ef1 --- /dev/null +++ b/TODO @@ -0,0 +1,6 @@ +- Validate input arguments: for example some ints must be >= 0 +- Add options to control the delay of when frames are added or removed from + the trail +- Some trivial cases of trails just need a framebuffer with no need to keep + the history of frames, see if the code handling those cases can be optimized +- Clean up the code? My C++ is a little C-ish... diff --git a/Trail.hpp b/Trail.hpp new file mode 100644 index 0000000..e519c94 --- /dev/null +++ b/Trail.hpp @@ -0,0 +1,180 @@ +#ifndef TRAIL_HPP +#define TRAIL_HPP + +#include +#include "Frame.hpp" + +typedef std::list trail_t; + +class Trail { +private: + virtual void iterate(cv::Mat& frame, void (*action)(cv::Mat&, Frame *, int, int)) = 0; + + cv::Mat *background = NULL; + bool redraw_current_frame; + bool persistent; + + void (*drawStrategy)(cv::Mat&, Frame *, int, int); + + static void frameCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + (void) trail_size; + frame->copyTo(destination); + } + + static void frameAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + (void) trail_size; + cv::accumulate(frame->getData(), destination, frame->getMask()); + } + + static void frameFadeCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + double weight = ((double) frame_index) / trail_size; + cv::Mat weighted_frame; + + frame->getData().convertTo(weighted_frame, destination.type(), weight); + weighted_frame.copyTo(destination, frame->getMask()); + } + + static void frameFadeAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + double weight = ((double) frame_index) / trail_size; + cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask()); + } + + static void frameAverage(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + double weight = 1. / trail_size; + cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask()); + } + +protected: + trail_t trail; + +public: + Trail(int trail_length, cv::Size frame_size) + { + redraw_current_frame = false; + drawStrategy = frameCopy; + persistent = false; + + if (trail_length < 0) { + persistent = true; + background = new cv::Mat(frame_size, CV_8UC3); + } + + if (trail_length < 1) + trail_length = 1; + + for (int i = 0; i < trail_length; i++) + trail.push_back(new Frame(frame_size)); + } + + virtual ~Trail() + { + trail_t::iterator trail_it; + for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) { + delete(*trail_it); + } + trail.clear(); + delete background; + } + + void setBackground(cv::Mat trail_background) + { + delete background; + background = new cv::Mat(trail_background); + } + + int setDrawingMethod(std::string drawing_method) + { + if (drawing_method == "copy") { + drawStrategy = frameCopy; + } else if (drawing_method == "accumulate") { + drawStrategy = frameAccumulate; + } else if (drawing_method == "fadecopy") { + drawStrategy = frameFadeCopy; + } else if (drawing_method == "fadeaccumulate") { + drawStrategy = frameFadeAccumulate; + } else if (drawing_method == "average") { + drawStrategy = frameAverage; + } else { + return -1; + } + return 0; + } + + void setRedrawCurrentFrame(bool do_redraw_current_frame) + { + redraw_current_frame = do_redraw_current_frame; + } + + void update(Frame *frame) + { + Frame *front = trail.front(); + trail.pop_front(); + delete front; + + trail.push_back(frame); + } + + void draw(cv::Mat& destination) + { + cv::Mat float_destination; + + if (background) + background->convertTo(float_destination, CV_32FC3); + else + destination.convertTo(float_destination, CV_32FC3); + + iterate(float_destination, drawStrategy); + float_destination.convertTo(destination, destination.type()); + + if (redraw_current_frame) { + Frame *current_frame = trail.back(); + current_frame->getData().copyTo(destination, current_frame->getMask()); + } + + /* remember the current trail in the background image */ + if (persistent) + float_destination.convertTo(*background, background->type()); + } +}; + +class ForwardTrail: public Trail { + void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int)) + { + int index = 0; + trail_t::iterator trail_it; + for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) { + action(destination, *trail_it, ++index, trail.size()); + } + } + +public: + ForwardTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size) + { + } +}; + +class ReverseTrail: public Trail { + void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int)) + { + int index = 0; + trail_t::reverse_iterator trail_it; + for (trail_it = trail.rbegin(); trail_it != trail.rend(); trail_it++) { + action(destination, *trail_it, ++index, trail.size()); + } + } + +public: + ReverseTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size) + { + } +}; + +#endif // TRAIL_HPP diff --git a/opencv_trail_effect.cpp b/opencv_trail_effect.cpp new file mode 100644 index 0000000..97b4aa7 --- /dev/null +++ b/opencv_trail_effect.cpp @@ -0,0 +1,226 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "Frame.hpp" +#include "Segmentation.hpp" +#include "Trail.hpp" + +static void usage(const char *name) +{ + std::cout << "usage: " << name << " [OPTIONS]" << std::endl; + std::cout << "OPTIONS:" << std::endl; + std::cout << "\t-i \tthe input file (if missing, a webcam will be tried)" << std::endl; + std::cout << "\t-o \tthe optional output file" << std::endl; + std::cout << "\t-l \tthe trail length in frames" << std::endl; + std::cout << "\t\t\tthe default is 25" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * a negative value means an 'infinite' trail" << std::endl; + std::cout << "\t-s \tthe image segmentation method" << std::endl; + std::cout << "\t\t\tvalid values are:" << std::endl; + std::cout << "\t\t\t none, threshold, background" << std::endl; + std::cout << "\t\t\tthe default is 'background'" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * 'none' is only useful with '-d average'" << std::endl; + std::cout << "\t-b \tthe number of initial frames for background learning," << std::endl; + std::cout << "\t\t\tthe default is 50" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only useful with '-s background'" << std::endl; + std::cout << "\t-t \tthe level for the threshold segmentation method," << std::endl; + std::cout << "\t\t\tthe default is 5" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only useful with '-s threshold'" << std::endl; + std::cout << "\t-d \tthe trail drawing method" << std::endl; + std::cout << "\t\t\tvalid values are:" << std::endl; + std::cout << "\t\t\t copy, accumulate, fadecopy, fadeaccumulate, average" << std::endl; + std::cout << "\t\t\tthe default is 'copy'" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * 'copy' is useless with '-s none'" << std::endl; + std::cout << "\t\t\t * the difference between 'fadecopy' and" << std::endl; + std::cout << "\t\t\t 'fadeaccumulate is cleared when using '-B'" << std::endl; + std::cout << "\t-r\t\treverse the trail drawing sequence" << std::endl; + std::cout << "\t-B\t\tshow the background behind the trail" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only used with '-s background'" << std::endl; + std::cout << "\t-F\t\tredraw the current frame on top of the trail" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * noticeable with '-s average'" << std::endl; + std::cout << "\t\t\t * noticeable with reverse faded trails" << std::endl; +} + +int main(int argc, char *argv[]) +{ + int ret = 0; + int opt; + + std::string *input_file = NULL; + std::string *output_file = NULL; + int trail_lenght = 25; + std::string *segmentation_method = new std::string("background"); + int background_learn_frames = 50; + int threshold_level = 5; + std::string *drawing_method = new std::string("copy"); + bool reverse_trail = false; + bool show_background = false; + bool redraw_current_frame = false; + + while ((opt = getopt(argc, argv, "i:o:l:s:b:t:d:rBFh")) != -1) { + switch (opt) { + case 'i': + input_file = new std::string(optarg); + break; + case 'o': + output_file = new std::string(optarg); + break; + case 'l': + trail_lenght = atoi(optarg); + break; + case 's': + delete segmentation_method; + segmentation_method = new std::string(optarg); + break; + case 'b': + background_learn_frames = atoi(optarg); + break; + case 't': + threshold_level = atoi(optarg); + break; + case 'd': + delete drawing_method; + drawing_method = new std::string(optarg); + break; + case 'r': + reverse_trail = true; + break; + case 'B': + show_background = true; + break; + case 'F': + redraw_current_frame = true; + break; + case 'h': + usage(argv[0]); + return 0; + default: /* '?' */ + usage(argv[0]); + return -1; + } + } + + cv::VideoCapture inputVideo; + cv::VideoWriter outputVideo; + cv::Size frame_size; + cv::Mat input_frame; + + if (input_file) { + inputVideo.open(*input_file); + } else { + inputVideo.open(0); + } + + if (!inputVideo.isOpened()) { + std::cerr << "Could not open the input video." << std::endl; + ret = -1; + goto out; + } + + frame_size = cv::Size((int) inputVideo.get(CV_CAP_PROP_FRAME_WIDTH), + (int) inputVideo.get(CV_CAP_PROP_FRAME_HEIGHT)); + + if (output_file) { + int fps = inputVideo.get(CV_CAP_PROP_FPS); + if (fps < 0) + fps = 25; + + outputVideo.open(*output_file, CV_FOURCC('M','J','P','G'), fps, frame_size, true); + if (!outputVideo.isOpened()) { + std::cerr << "Could not open the output video for write." << std::endl; + ret = -1; + goto out; + } + } + + Trail *trail; + if (reverse_trail) + trail = new ReverseTrail(trail_lenght, frame_size); + else + trail = new ForwardTrail(trail_lenght, frame_size); + + trail->setRedrawCurrentFrame(redraw_current_frame); + + if (trail->setDrawingMethod(*drawing_method) < 0) { + std::cerr << "Invalid drawing method." << std::endl; + ret = -1; + goto out_delete_trail; + } + + Segmentation *segmentation; + if (*segmentation_method == "background") { + segmentation = new MOG2Segmentation(inputVideo, background_learn_frames); + if (show_background) { + cv::Mat background; + + ((MOG2Segmentation *)segmentation)->getBackgroundImage(background); + trail->setBackground(background); + } + } else if (*segmentation_method == "threshold") { + segmentation = new ThresholdSegmentation(threshold_level); + } else if (*segmentation_method == "none") { + segmentation = new DummySegmentation(); + } else { + std::cerr << "Invalid segmentation method." << std::endl; + ret = -1; + goto out_delete_trail; + } + + cv::namedWindow("Frame", CV_WINDOW_AUTOSIZE); + + for (;;) { + inputVideo >> input_frame; + + Frame *foreground = new Frame(input_frame, + segmentation->getForegroundMask(input_frame)); + trail->update(foreground); + + cv::Mat canvas = cv::Mat::zeros(input_frame.size(), input_frame.type()); + trail->draw(canvas); + + cv::imshow("Frame", canvas); + if (cv::waitKey(30) >= 0) + break; + + if (outputVideo.isOpened()) + outputVideo << canvas; + } + + cv::destroyWindow("Frame"); + + delete segmentation; + +out_delete_trail: + delete trail; +out: + delete drawing_method; + delete segmentation_method; + delete output_file; + delete input_file; + return ret; +}