From d095022ca7b1d9eb46521073b71f2327fb9d5637 Mon Sep 17 00:00:00 2001 From: Antonio Ospite Date: Mon, 28 Sep 2015 23:21:46 +0200 Subject: [PATCH] Initial import --- .gitignore | 1 + Frame.hpp | 78 +++++++++++++++++ Makefile | 56 ++++++++++++ README | 63 ++++++++++++++ Segmentation.hpp | 94 ++++++++++++++++++++ TODO | 6 ++ Trail.hpp | 180 ++++++++++++++++++++++++++++++++++++++ opencv_trail_effect.cpp | 226 ++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 704 insertions(+) create mode 100644 .gitignore create mode 100644 Frame.hpp create mode 100644 Makefile create mode 100644 README create mode 100644 Segmentation.hpp create mode 100644 TODO create mode 100644 Trail.hpp create mode 100644 opencv_trail_effect.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75be08c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +opencv_trail_effect diff --git a/Frame.hpp b/Frame.hpp new file mode 100644 index 0000000..662954b --- /dev/null +++ b/Frame.hpp @@ -0,0 +1,78 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef FRAME_HPP +#define FRAME_HPP + +#include + +class Frame { +private: + cv::Mat data; + cv::Mat mask; + +public: + Frame(cv::Size size) + { + data = cv::Mat::zeros(size, CV_8UC3); + mask = cv::Mat::zeros(size, CV_8UC1); + } + + Frame(const cv::Mat& frame_data) + { + frame_data.copyTo(data); + } + + Frame(const cv::Mat& frame_data, const cv::Mat& frame_mask) + { + frame_data.copyTo(data); + frame_mask.copyTo(mask); + } + + ~Frame() + { + } + + void copyTo(cv::Mat& destination) + { + cv::Mat converted_data; + + data.convertTo(converted_data, destination.type()); + converted_data.copyTo(destination, mask); + } + + void zero() + { + data.setTo(cv::Scalar(0)); + mask.setTo(cv::Scalar(0)); + } + + cv::Mat getMasked() + { + cv::Mat masked_frame; + + data.copyTo(masked_frame, mask); + return masked_frame; + } + + cv::Mat getData() { return data; } + cv::Mat getMask() { return mask; } +}; + +#endif // FRAME_HPP diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..de166de --- /dev/null +++ b/Makefile @@ -0,0 +1,56 @@ +CXXFLAGS = -ansi -pedantic -pedantic-errors -Wall -g3 -O2 -D_ANSI_SOURCE_ +CXXFLAGS += -fno-common \ + -fvisibility=hidden \ + -Wall \ + -Wextra \ + -Wformat=2 \ + -Winit-self \ + -Winline \ + -Wpacked \ + -Wpointer-arith \ + -Wlarger-than-65500 \ + -Wmissing-declarations \ + -Wmissing-format-attribute \ + -Wmissing-noreturn \ + -Wredundant-decls \ + -Wsign-compare \ + -Wstrict-aliasing=2 \ + -Wswitch-enum \ + -Wundef \ + -Wunreachable-code \ + -Wunsafe-loop-optimizations \ + -Wunused-but-set-variable \ + -Wwrite-strings \ + -Wp,-D_FORTIFY_SOURCE=2 \ + -fstack-protector \ + --param=ssp-buffer-size=4 + +CXXFLAGS += $(shell pkg-config --cflags opencv) +LDLIBS += $(shell pkg-config --libs opencv) + +opencv_trail_effect: $(wildcard *.hpp) + + +# Some command lines to imitate different trail styles + +blame_it_on_the_boogie: opencv_trail_effect + ./opencv_trail_effect -l 12 -s background -d fadeaccumulate + +wtf: opencv_trail_effect + ./opencv_trail_effect -l -1 -s background -d copy + +l_anima_vola: opencv_trail_effect + ./opencv_trail_effect -l 30 -s background -d copy -r + +average: opencv_trail_effect + ./opencv_trail_effect -l 10 -s background -d average -B + +clean: + rm -f opencv_trail_effect vgdump gtk.suppression + +test: opencv_trail_effect + [ -f gtk.suppression ] || wget -nv https://people.gnome.org/~johan/gtk.suppression + G_DEBUG=gc-friendly G_SLICE=always-malloc \ + valgrind --tool=memcheck --leak-check=full --leak-resolution=high \ + --num-callers=20 --log-file=vgdump \ + --suppressions=gtk.suppression ./opencv_trail_effect diff --git a/README b/README new file mode 100644 index 0000000..cd242b0 --- /dev/null +++ b/README @@ -0,0 +1,63 @@ +opencv_trail_effect is an experiment about recreating the "trail effect"[1], +which can be seen in some cool music videos, automatically and in real-time, +and even without the use of a green-screen. + +[1] http://dailypsychedelicvideo.com/tag/trail-effect/ + +Some examples of it are: + + The Jacksons - Blame It On The Boogie - https://www.youtube.com/watch?v=mkBS4zUjJZo + OK Go - WTF? - https://www.youtube.com/watch?v=12zJw9varYE + Elisa - L'Anima Vola - https://www.youtube.com/watch?v=MqhxIQD16EA + +opencv_trail_effect draws a trail after performing foreground segmentation +(using thresholding or background subtraction) using OpenCV. + +The "sole" assumptions for a decent effect are that: + + 1. the camera does not move; + 2. the background lighting in the scene is quite stable (when doing + background subtraction it is recommended to disable auto-gain in the + camera, and avoiding any artifact introduced by the power line + frequency). + + +Examples of use +--------------- + +The effect in "Blame It On The Boogie" could be described as a short faded +trail and can be achieved with this command line: + + $ ./opencv_trail_effect -l 12 -s background -d fadeaccumulate + + +The effect in "WTF?" is equivalent to an infinite trail in which the isolated +foreground is copied on the background and stays there, it can be achieved +with this command line: + + $ ./opencv_trail_effect -l -1 -s background -d copy + + +The effect in "L'Anima Vola" seems to have a "catch-up" behavior; that one can +be done by drawing the trail in reverse, with this command line: + + $ ./opencv_trail_effect -l 30 -s background -d copy -r + + +Another effect seen in movies can be obtained by doing the average of the +frames in the trail, possibly with the most recent frame drawn on top: + + $ ./opencv_trail_effect -l 10 -s background -d average -B + + +Side note +------------------- + +These effects can bring to mind other cool effects in music videos, like the +frame shuffling effect in Peter Gabriel's Sledgehammer +(https://www.youtube.com/watch?v=OJWJE0x7T4Q), that one can be done with the +nervousTV plugin from effectv +(https://fukuchi.org/research/effectv/index.html.en) or frei0r filters +(https://www.dyne.org/software/frei0r/), for example with: + + $ gst-launch-1.0 -v v4l2src ! videoconvert ! frei0r-filter-nervous ! videoconvert ! autovideosink diff --git a/Segmentation.hpp b/Segmentation.hpp new file mode 100644 index 0000000..bd9b4c9 --- /dev/null +++ b/Segmentation.hpp @@ -0,0 +1,94 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef SEGMENTATION_HPP +#define SEGMENTATION_HPP + +#include + +class Segmentation { +public: + virtual cv::Mat getForegroundMask(const cv::Mat& frame) = 0; + virtual ~Segmentation() {} +}; + +class DummySegmentation : public Segmentation { +public: + cv::Mat getForegroundMask(const cv::Mat& frame) + { + return cv::Mat(frame.size(), CV_8UC1, cv::Scalar(1)); + } +}; + +class ThresholdSegmentation : public Segmentation { +private: + int threshold; + +public: + ThresholdSegmentation(int threshold_value) + { + threshold = threshold_value; + } + + cv::Mat getForegroundMask(const cv::Mat& frame) + { + cv::Mat gray_frame; + cv::Mat frame_mask; + + cvtColor(frame, gray_frame, CV_RGB2GRAY); + cv::threshold(gray_frame, frame_mask, threshold, 255, CV_THRESH_TOZERO); + + return frame_mask; + } +}; + +/* + * Some background on... background subtraction can be found here: + * http://docs.opencv.org/master/d1/dc5/tutorial_background_subtraction.html + */ + +class MOG2Segmentation : public Segmentation, public cv::BackgroundSubtractorMOG2 { +public: + MOG2Segmentation(cv::VideoCapture& inputVideo, int learning_frames) : + cv::BackgroundSubtractorMOG2() + { + cv::Mat background; + cv::Mat foreground_mask; + + for (int i = 0; i < learning_frames; i++) { + inputVideo >> background; + this->operator()(background, foreground_mask); + } + } + + cv::Mat getForegroundMask(const cv::Mat& frame) + { + cv::Mat foreground_mask; + + this->operator()(frame, foreground_mask, 0); + cv::erode(foreground_mask, foreground_mask, cv::Mat()); + cv::dilate(foreground_mask, foreground_mask, cv::Mat()); + cv::threshold(foreground_mask, foreground_mask, 0, 255, CV_THRESH_OTSU); + cv::medianBlur(foreground_mask, foreground_mask, 9); + + return foreground_mask; + } +}; + +#endif // SEGMENTATION_HPP diff --git a/TODO b/TODO new file mode 100644 index 0000000..8aa8ef1 --- /dev/null +++ b/TODO @@ -0,0 +1,6 @@ +- Validate input arguments: for example some ints must be >= 0 +- Add options to control the delay of when frames are added or removed from + the trail +- Some trivial cases of trails just need a framebuffer with no need to keep + the history of frames, see if the code handling those cases can be optimized +- Clean up the code? My C++ is a little C-ish... diff --git a/Trail.hpp b/Trail.hpp new file mode 100644 index 0000000..e519c94 --- /dev/null +++ b/Trail.hpp @@ -0,0 +1,180 @@ +#ifndef TRAIL_HPP +#define TRAIL_HPP + +#include +#include "Frame.hpp" + +typedef std::list trail_t; + +class Trail { +private: + virtual void iterate(cv::Mat& frame, void (*action)(cv::Mat&, Frame *, int, int)) = 0; + + cv::Mat *background = NULL; + bool redraw_current_frame; + bool persistent; + + void (*drawStrategy)(cv::Mat&, Frame *, int, int); + + static void frameCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + (void) trail_size; + frame->copyTo(destination); + } + + static void frameAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + (void) trail_size; + cv::accumulate(frame->getData(), destination, frame->getMask()); + } + + static void frameFadeCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + double weight = ((double) frame_index) / trail_size; + cv::Mat weighted_frame; + + frame->getData().convertTo(weighted_frame, destination.type(), weight); + weighted_frame.copyTo(destination, frame->getMask()); + } + + static void frameFadeAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + double weight = ((double) frame_index) / trail_size; + cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask()); + } + + static void frameAverage(cv::Mat& destination, Frame *frame, int frame_index, int trail_size) + { + (void) frame_index; + double weight = 1. / trail_size; + cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask()); + } + +protected: + trail_t trail; + +public: + Trail(int trail_length, cv::Size frame_size) + { + redraw_current_frame = false; + drawStrategy = frameCopy; + persistent = false; + + if (trail_length < 0) { + persistent = true; + background = new cv::Mat(frame_size, CV_8UC3); + } + + if (trail_length < 1) + trail_length = 1; + + for (int i = 0; i < trail_length; i++) + trail.push_back(new Frame(frame_size)); + } + + virtual ~Trail() + { + trail_t::iterator trail_it; + for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) { + delete(*trail_it); + } + trail.clear(); + delete background; + } + + void setBackground(cv::Mat trail_background) + { + delete background; + background = new cv::Mat(trail_background); + } + + int setDrawingMethod(std::string drawing_method) + { + if (drawing_method == "copy") { + drawStrategy = frameCopy; + } else if (drawing_method == "accumulate") { + drawStrategy = frameAccumulate; + } else if (drawing_method == "fadecopy") { + drawStrategy = frameFadeCopy; + } else if (drawing_method == "fadeaccumulate") { + drawStrategy = frameFadeAccumulate; + } else if (drawing_method == "average") { + drawStrategy = frameAverage; + } else { + return -1; + } + return 0; + } + + void setRedrawCurrentFrame(bool do_redraw_current_frame) + { + redraw_current_frame = do_redraw_current_frame; + } + + void update(Frame *frame) + { + Frame *front = trail.front(); + trail.pop_front(); + delete front; + + trail.push_back(frame); + } + + void draw(cv::Mat& destination) + { + cv::Mat float_destination; + + if (background) + background->convertTo(float_destination, CV_32FC3); + else + destination.convertTo(float_destination, CV_32FC3); + + iterate(float_destination, drawStrategy); + float_destination.convertTo(destination, destination.type()); + + if (redraw_current_frame) { + Frame *current_frame = trail.back(); + current_frame->getData().copyTo(destination, current_frame->getMask()); + } + + /* remember the current trail in the background image */ + if (persistent) + float_destination.convertTo(*background, background->type()); + } +}; + +class ForwardTrail: public Trail { + void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int)) + { + int index = 0; + trail_t::iterator trail_it; + for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) { + action(destination, *trail_it, ++index, trail.size()); + } + } + +public: + ForwardTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size) + { + } +}; + +class ReverseTrail: public Trail { + void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int)) + { + int index = 0; + trail_t::reverse_iterator trail_it; + for (trail_it = trail.rbegin(); trail_it != trail.rend(); trail_it++) { + action(destination, *trail_it, ++index, trail.size()); + } + } + +public: + ReverseTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size) + { + } +}; + +#endif // TRAIL_HPP diff --git a/opencv_trail_effect.cpp b/opencv_trail_effect.cpp new file mode 100644 index 0000000..97b4aa7 --- /dev/null +++ b/opencv_trail_effect.cpp @@ -0,0 +1,226 @@ +/* + * openvc_trail_effect - experiments about video trail effects + * + * Copyright (C) 2015 Antonio Ospite + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "Frame.hpp" +#include "Segmentation.hpp" +#include "Trail.hpp" + +static void usage(const char *name) +{ + std::cout << "usage: " << name << " [OPTIONS]" << std::endl; + std::cout << "OPTIONS:" << std::endl; + std::cout << "\t-i \tthe input file (if missing, a webcam will be tried)" << std::endl; + std::cout << "\t-o \tthe optional output file" << std::endl; + std::cout << "\t-l \tthe trail length in frames" << std::endl; + std::cout << "\t\t\tthe default is 25" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * a negative value means an 'infinite' trail" << std::endl; + std::cout << "\t-s \tthe image segmentation method" << std::endl; + std::cout << "\t\t\tvalid values are:" << std::endl; + std::cout << "\t\t\t none, threshold, background" << std::endl; + std::cout << "\t\t\tthe default is 'background'" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * 'none' is only useful with '-d average'" << std::endl; + std::cout << "\t-b \tthe number of initial frames for background learning," << std::endl; + std::cout << "\t\t\tthe default is 50" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only useful with '-s background'" << std::endl; + std::cout << "\t-t \tthe level for the threshold segmentation method," << std::endl; + std::cout << "\t\t\tthe default is 5" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only useful with '-s threshold'" << std::endl; + std::cout << "\t-d \tthe trail drawing method" << std::endl; + std::cout << "\t\t\tvalid values are:" << std::endl; + std::cout << "\t\t\t copy, accumulate, fadecopy, fadeaccumulate, average" << std::endl; + std::cout << "\t\t\tthe default is 'copy'" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * 'copy' is useless with '-s none'" << std::endl; + std::cout << "\t\t\t * the difference between 'fadecopy' and" << std::endl; + std::cout << "\t\t\t 'fadeaccumulate is cleared when using '-B'" << std::endl; + std::cout << "\t-r\t\treverse the trail drawing sequence" << std::endl; + std::cout << "\t-B\t\tshow the background behind the trail" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * only used with '-s background'" << std::endl; + std::cout << "\t-F\t\tredraw the current frame on top of the trail" << std::endl; + std::cout << "\t\t\tNOTES:" << std::endl; + std::cout << "\t\t\t * noticeable with '-s average'" << std::endl; + std::cout << "\t\t\t * noticeable with reverse faded trails" << std::endl; +} + +int main(int argc, char *argv[]) +{ + int ret = 0; + int opt; + + std::string *input_file = NULL; + std::string *output_file = NULL; + int trail_lenght = 25; + std::string *segmentation_method = new std::string("background"); + int background_learn_frames = 50; + int threshold_level = 5; + std::string *drawing_method = new std::string("copy"); + bool reverse_trail = false; + bool show_background = false; + bool redraw_current_frame = false; + + while ((opt = getopt(argc, argv, "i:o:l:s:b:t:d:rBFh")) != -1) { + switch (opt) { + case 'i': + input_file = new std::string(optarg); + break; + case 'o': + output_file = new std::string(optarg); + break; + case 'l': + trail_lenght = atoi(optarg); + break; + case 's': + delete segmentation_method; + segmentation_method = new std::string(optarg); + break; + case 'b': + background_learn_frames = atoi(optarg); + break; + case 't': + threshold_level = atoi(optarg); + break; + case 'd': + delete drawing_method; + drawing_method = new std::string(optarg); + break; + case 'r': + reverse_trail = true; + break; + case 'B': + show_background = true; + break; + case 'F': + redraw_current_frame = true; + break; + case 'h': + usage(argv[0]); + return 0; + default: /* '?' */ + usage(argv[0]); + return -1; + } + } + + cv::VideoCapture inputVideo; + cv::VideoWriter outputVideo; + cv::Size frame_size; + cv::Mat input_frame; + + if (input_file) { + inputVideo.open(*input_file); + } else { + inputVideo.open(0); + } + + if (!inputVideo.isOpened()) { + std::cerr << "Could not open the input video." << std::endl; + ret = -1; + goto out; + } + + frame_size = cv::Size((int) inputVideo.get(CV_CAP_PROP_FRAME_WIDTH), + (int) inputVideo.get(CV_CAP_PROP_FRAME_HEIGHT)); + + if (output_file) { + int fps = inputVideo.get(CV_CAP_PROP_FPS); + if (fps < 0) + fps = 25; + + outputVideo.open(*output_file, CV_FOURCC('M','J','P','G'), fps, frame_size, true); + if (!outputVideo.isOpened()) { + std::cerr << "Could not open the output video for write." << std::endl; + ret = -1; + goto out; + } + } + + Trail *trail; + if (reverse_trail) + trail = new ReverseTrail(trail_lenght, frame_size); + else + trail = new ForwardTrail(trail_lenght, frame_size); + + trail->setRedrawCurrentFrame(redraw_current_frame); + + if (trail->setDrawingMethod(*drawing_method) < 0) { + std::cerr << "Invalid drawing method." << std::endl; + ret = -1; + goto out_delete_trail; + } + + Segmentation *segmentation; + if (*segmentation_method == "background") { + segmentation = new MOG2Segmentation(inputVideo, background_learn_frames); + if (show_background) { + cv::Mat background; + + ((MOG2Segmentation *)segmentation)->getBackgroundImage(background); + trail->setBackground(background); + } + } else if (*segmentation_method == "threshold") { + segmentation = new ThresholdSegmentation(threshold_level); + } else if (*segmentation_method == "none") { + segmentation = new DummySegmentation(); + } else { + std::cerr << "Invalid segmentation method." << std::endl; + ret = -1; + goto out_delete_trail; + } + + cv::namedWindow("Frame", CV_WINDOW_AUTOSIZE); + + for (;;) { + inputVideo >> input_frame; + + Frame *foreground = new Frame(input_frame, + segmentation->getForegroundMask(input_frame)); + trail->update(foreground); + + cv::Mat canvas = cv::Mat::zeros(input_frame.size(), input_frame.type()); + trail->draw(canvas); + + cv::imshow("Frame", canvas); + if (cv::waitKey(30) >= 0) + break; + + if (outputVideo.isOpened()) + outputVideo << canvas; + } + + cv::destroyWindow("Frame"); + + delete segmentation; + +out_delete_trail: + delete trail; +out: + delete drawing_method; + delete segmentation_method; + delete output_file; + delete input_file; + return ret; +} -- 2.1.4