--- /dev/null
+opencv_trail_effect
--- /dev/null
+/*
+ * openvc_trail_effect - experiments about video trail effects
+ *
+ * Copyright (C) 2015 Antonio Ospite <ao2@ao2.it>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef FRAME_HPP
+#define FRAME_HPP
+
+#include <opencv2/opencv.hpp>
+
+class Frame {
+private:
+ cv::Mat data;
+ cv::Mat mask;
+
+public:
+ Frame(cv::Size size)
+ {
+ data = cv::Mat::zeros(size, CV_8UC3);
+ mask = cv::Mat::zeros(size, CV_8UC1);
+ }
+
+ Frame(const cv::Mat& frame_data)
+ {
+ frame_data.copyTo(data);
+ }
+
+ Frame(const cv::Mat& frame_data, const cv::Mat& frame_mask)
+ {
+ frame_data.copyTo(data);
+ frame_mask.copyTo(mask);
+ }
+
+ ~Frame()
+ {
+ }
+
+ void copyTo(cv::Mat& destination)
+ {
+ cv::Mat converted_data;
+
+ data.convertTo(converted_data, destination.type());
+ converted_data.copyTo(destination, mask);
+ }
+
+ void zero()
+ {
+ data.setTo(cv::Scalar(0));
+ mask.setTo(cv::Scalar(0));
+ }
+
+ cv::Mat getMasked()
+ {
+ cv::Mat masked_frame;
+
+ data.copyTo(masked_frame, mask);
+ return masked_frame;
+ }
+
+ cv::Mat getData() { return data; }
+ cv::Mat getMask() { return mask; }
+};
+
+#endif // FRAME_HPP
--- /dev/null
+CXXFLAGS = -ansi -pedantic -pedantic-errors -Wall -g3 -O2 -D_ANSI_SOURCE_
+CXXFLAGS += -fno-common \
+ -fvisibility=hidden \
+ -Wall \
+ -Wextra \
+ -Wformat=2 \
+ -Winit-self \
+ -Winline \
+ -Wpacked \
+ -Wpointer-arith \
+ -Wlarger-than-65500 \
+ -Wmissing-declarations \
+ -Wmissing-format-attribute \
+ -Wmissing-noreturn \
+ -Wredundant-decls \
+ -Wsign-compare \
+ -Wstrict-aliasing=2 \
+ -Wswitch-enum \
+ -Wundef \
+ -Wunreachable-code \
+ -Wunsafe-loop-optimizations \
+ -Wunused-but-set-variable \
+ -Wwrite-strings \
+ -Wp,-D_FORTIFY_SOURCE=2 \
+ -fstack-protector \
+ --param=ssp-buffer-size=4
+
+CXXFLAGS += $(shell pkg-config --cflags opencv)
+LDLIBS += $(shell pkg-config --libs opencv)
+
+opencv_trail_effect: $(wildcard *.hpp)
+
+
+# Some command lines to imitate different trail styles
+
+blame_it_on_the_boogie: opencv_trail_effect
+ ./opencv_trail_effect -l 12 -s background -d fadeaccumulate
+
+wtf: opencv_trail_effect
+ ./opencv_trail_effect -l -1 -s background -d copy
+
+l_anima_vola: opencv_trail_effect
+ ./opencv_trail_effect -l 30 -s background -d copy -r
+
+average: opencv_trail_effect
+ ./opencv_trail_effect -l 10 -s background -d average -B
+
+clean:
+ rm -f opencv_trail_effect vgdump gtk.suppression
+
+test: opencv_trail_effect
+ [ -f gtk.suppression ] || wget -nv https://people.gnome.org/~johan/gtk.suppression
+ G_DEBUG=gc-friendly G_SLICE=always-malloc \
+ valgrind --tool=memcheck --leak-check=full --leak-resolution=high \
+ --num-callers=20 --log-file=vgdump \
+ --suppressions=gtk.suppression ./opencv_trail_effect
--- /dev/null
+opencv_trail_effect is an experiment about recreating the "trail effect"[1],
+which can be seen in some cool music videos, automatically and in real-time,
+and even without the use of a green-screen.
+
+[1] http://dailypsychedelicvideo.com/tag/trail-effect/
+
+Some examples of it are:
+
+ The Jacksons - Blame It On The Boogie - https://www.youtube.com/watch?v=mkBS4zUjJZo
+ OK Go - WTF? - https://www.youtube.com/watch?v=12zJw9varYE
+ Elisa - L'Anima Vola - https://www.youtube.com/watch?v=MqhxIQD16EA
+
+opencv_trail_effect draws a trail after performing foreground segmentation
+(using thresholding or background subtraction) using OpenCV.
+
+The "sole" assumptions for a decent effect are that:
+
+ 1. the camera does not move;
+ 2. the background lighting in the scene is quite stable (when doing
+ background subtraction it is recommended to disable auto-gain in the
+ camera, and avoiding any artifact introduced by the power line
+ frequency).
+
+
+Examples of use
+---------------
+
+The effect in "Blame It On The Boogie" could be described as a short faded
+trail and can be achieved with this command line:
+
+ $ ./opencv_trail_effect -l 12 -s background -d fadeaccumulate
+
+
+The effect in "WTF?" is equivalent to an infinite trail in which the isolated
+foreground is copied on the background and stays there, it can be achieved
+with this command line:
+
+ $ ./opencv_trail_effect -l -1 -s background -d copy
+
+
+The effect in "L'Anima Vola" seems to have a "catch-up" behavior; that one can
+be done by drawing the trail in reverse, with this command line:
+
+ $ ./opencv_trail_effect -l 30 -s background -d copy -r
+
+
+Another effect seen in movies can be obtained by doing the average of the
+frames in the trail, possibly with the most recent frame drawn on top:
+
+ $ ./opencv_trail_effect -l 10 -s background -d average -B
+
+
+Side note
+-------------------
+
+These effects can bring to mind other cool effects in music videos, like the
+frame shuffling effect in Peter Gabriel's Sledgehammer
+(https://www.youtube.com/watch?v=OJWJE0x7T4Q), that one can be done with the
+nervousTV plugin from effectv
+(https://fukuchi.org/research/effectv/index.html.en) or frei0r filters
+(https://www.dyne.org/software/frei0r/), for example with:
+
+ $ gst-launch-1.0 -v v4l2src ! videoconvert ! frei0r-filter-nervous ! videoconvert ! autovideosink
--- /dev/null
+/*
+ * openvc_trail_effect - experiments about video trail effects
+ *
+ * Copyright (C) 2015 Antonio Ospite <ao2@ao2.it>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SEGMENTATION_HPP
+#define SEGMENTATION_HPP
+
+#include <opencv2/opencv.hpp>
+
+class Segmentation {
+public:
+ virtual cv::Mat getForegroundMask(const cv::Mat& frame) = 0;
+ virtual ~Segmentation() {}
+};
+
+class DummySegmentation : public Segmentation {
+public:
+ cv::Mat getForegroundMask(const cv::Mat& frame)
+ {
+ return cv::Mat(frame.size(), CV_8UC1, cv::Scalar(1));
+ }
+};
+
+class ThresholdSegmentation : public Segmentation {
+private:
+ int threshold;
+
+public:
+ ThresholdSegmentation(int threshold_value)
+ {
+ threshold = threshold_value;
+ }
+
+ cv::Mat getForegroundMask(const cv::Mat& frame)
+ {
+ cv::Mat gray_frame;
+ cv::Mat frame_mask;
+
+ cvtColor(frame, gray_frame, CV_RGB2GRAY);
+ cv::threshold(gray_frame, frame_mask, threshold, 255, CV_THRESH_TOZERO);
+
+ return frame_mask;
+ }
+};
+
+/*
+ * Some background on... background subtraction can be found here:
+ * http://docs.opencv.org/master/d1/dc5/tutorial_background_subtraction.html
+ */
+
+class MOG2Segmentation : public Segmentation, public cv::BackgroundSubtractorMOG2 {
+public:
+ MOG2Segmentation(cv::VideoCapture& inputVideo, int learning_frames) :
+ cv::BackgroundSubtractorMOG2()
+ {
+ cv::Mat background;
+ cv::Mat foreground_mask;
+
+ for (int i = 0; i < learning_frames; i++) {
+ inputVideo >> background;
+ this->operator()(background, foreground_mask);
+ }
+ }
+
+ cv::Mat getForegroundMask(const cv::Mat& frame)
+ {
+ cv::Mat foreground_mask;
+
+ this->operator()(frame, foreground_mask, 0);
+ cv::erode(foreground_mask, foreground_mask, cv::Mat());
+ cv::dilate(foreground_mask, foreground_mask, cv::Mat());
+ cv::threshold(foreground_mask, foreground_mask, 0, 255, CV_THRESH_OTSU);
+ cv::medianBlur(foreground_mask, foreground_mask, 9);
+
+ return foreground_mask;
+ }
+};
+
+#endif // SEGMENTATION_HPP
--- /dev/null
+- Validate input arguments: for example some ints must be >= 0
+- Add options to control the delay of when frames are added or removed from
+ the trail
+- Some trivial cases of trails just need a framebuffer with no need to keep
+ the history of frames, see if the code handling those cases can be optimized
+- Clean up the code? My C++ is a little C-ish...
--- /dev/null
+#ifndef TRAIL_HPP
+#define TRAIL_HPP
+
+#include <opencv2/opencv.hpp>
+#include "Frame.hpp"
+
+typedef std::list<Frame *> trail_t;
+
+class Trail {
+private:
+ virtual void iterate(cv::Mat& frame, void (*action)(cv::Mat&, Frame *, int, int)) = 0;
+
+ cv::Mat *background = NULL;
+ bool redraw_current_frame;
+ bool persistent;
+
+ void (*drawStrategy)(cv::Mat&, Frame *, int, int);
+
+ static void frameCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size)
+ {
+ (void) frame_index;
+ (void) trail_size;
+ frame->copyTo(destination);
+ }
+
+ static void frameAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size)
+ {
+ (void) frame_index;
+ (void) trail_size;
+ cv::accumulate(frame->getData(), destination, frame->getMask());
+ }
+
+ static void frameFadeCopy(cv::Mat& destination, Frame *frame, int frame_index, int trail_size)
+ {
+ double weight = ((double) frame_index) / trail_size;
+ cv::Mat weighted_frame;
+
+ frame->getData().convertTo(weighted_frame, destination.type(), weight);
+ weighted_frame.copyTo(destination, frame->getMask());
+ }
+
+ static void frameFadeAccumulate(cv::Mat& destination, Frame *frame, int frame_index, int trail_size)
+ {
+ double weight = ((double) frame_index) / trail_size;
+ cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask());
+ }
+
+ static void frameAverage(cv::Mat& destination, Frame *frame, int frame_index, int trail_size)
+ {
+ (void) frame_index;
+ double weight = 1. / trail_size;
+ cv::accumulateWeighted(frame->getData(), destination, weight, frame->getMask());
+ }
+
+protected:
+ trail_t trail;
+
+public:
+ Trail(int trail_length, cv::Size frame_size)
+ {
+ redraw_current_frame = false;
+ drawStrategy = frameCopy;
+ persistent = false;
+
+ if (trail_length < 0) {
+ persistent = true;
+ background = new cv::Mat(frame_size, CV_8UC3);
+ }
+
+ if (trail_length < 1)
+ trail_length = 1;
+
+ for (int i = 0; i < trail_length; i++)
+ trail.push_back(new Frame(frame_size));
+ }
+
+ virtual ~Trail()
+ {
+ trail_t::iterator trail_it;
+ for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) {
+ delete(*trail_it);
+ }
+ trail.clear();
+ delete background;
+ }
+
+ void setBackground(cv::Mat trail_background)
+ {
+ delete background;
+ background = new cv::Mat(trail_background);
+ }
+
+ int setDrawingMethod(std::string drawing_method)
+ {
+ if (drawing_method == "copy") {
+ drawStrategy = frameCopy;
+ } else if (drawing_method == "accumulate") {
+ drawStrategy = frameAccumulate;
+ } else if (drawing_method == "fadecopy") {
+ drawStrategy = frameFadeCopy;
+ } else if (drawing_method == "fadeaccumulate") {
+ drawStrategy = frameFadeAccumulate;
+ } else if (drawing_method == "average") {
+ drawStrategy = frameAverage;
+ } else {
+ return -1;
+ }
+ return 0;
+ }
+
+ void setRedrawCurrentFrame(bool do_redraw_current_frame)
+ {
+ redraw_current_frame = do_redraw_current_frame;
+ }
+
+ void update(Frame *frame)
+ {
+ Frame *front = trail.front();
+ trail.pop_front();
+ delete front;
+
+ trail.push_back(frame);
+ }
+
+ void draw(cv::Mat& destination)
+ {
+ cv::Mat float_destination;
+
+ if (background)
+ background->convertTo(float_destination, CV_32FC3);
+ else
+ destination.convertTo(float_destination, CV_32FC3);
+
+ iterate(float_destination, drawStrategy);
+ float_destination.convertTo(destination, destination.type());
+
+ if (redraw_current_frame) {
+ Frame *current_frame = trail.back();
+ current_frame->getData().copyTo(destination, current_frame->getMask());
+ }
+
+ /* remember the current trail in the background image */
+ if (persistent)
+ float_destination.convertTo(*background, background->type());
+ }
+};
+
+class ForwardTrail: public Trail {
+ void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int))
+ {
+ int index = 0;
+ trail_t::iterator trail_it;
+ for (trail_it = trail.begin(); trail_it != trail.end(); trail_it++) {
+ action(destination, *trail_it, ++index, trail.size());
+ }
+ }
+
+public:
+ ForwardTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size)
+ {
+ }
+};
+
+class ReverseTrail: public Trail {
+ void iterate(cv::Mat& destination, void (*action)(cv::Mat&, Frame *, int, int))
+ {
+ int index = 0;
+ trail_t::reverse_iterator trail_it;
+ for (trail_it = trail.rbegin(); trail_it != trail.rend(); trail_it++) {
+ action(destination, *trail_it, ++index, trail.size());
+ }
+ }
+
+public:
+ ReverseTrail(int trail_lenght, cv::Size frame_size) : Trail(trail_lenght, frame_size)
+ {
+ }
+};
+
+#endif // TRAIL_HPP
--- /dev/null
+/*
+ * openvc_trail_effect - experiments about video trail effects
+ *
+ * Copyright (C) 2015 Antonio Ospite <ao2@ao2.it>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <opencv2/opencv.hpp>
+#include <unistd.h>
+
+#include "Frame.hpp"
+#include "Segmentation.hpp"
+#include "Trail.hpp"
+
+static void usage(const char *name)
+{
+ std::cout << "usage: " << name << " [OPTIONS]" << std::endl;
+ std::cout << "OPTIONS:" << std::endl;
+ std::cout << "\t-i <file>\tthe input file (if missing, a webcam will be tried)" << std::endl;
+ std::cout << "\t-o <file>\tthe optional output file" << std::endl;
+ std::cout << "\t-l <number>\tthe trail length in frames" << std::endl;
+ std::cout << "\t\t\tthe default is 25" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * a negative value means an 'infinite' trail" << std::endl;
+ std::cout << "\t-s <method>\tthe image segmentation method" << std::endl;
+ std::cout << "\t\t\tvalid values are:" << std::endl;
+ std::cout << "\t\t\t none, threshold, background" << std::endl;
+ std::cout << "\t\t\tthe default is 'background'" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * 'none' is only useful with '-d average'" << std::endl;
+ std::cout << "\t-b <number>\tthe number of initial frames for background learning," << std::endl;
+ std::cout << "\t\t\tthe default is 50" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * only useful with '-s background'" << std::endl;
+ std::cout << "\t-t <number>\tthe level for the threshold segmentation method," << std::endl;
+ std::cout << "\t\t\tthe default is 5" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * only useful with '-s threshold'" << std::endl;
+ std::cout << "\t-d <method>\tthe trail drawing method" << std::endl;
+ std::cout << "\t\t\tvalid values are:" << std::endl;
+ std::cout << "\t\t\t copy, accumulate, fadecopy, fadeaccumulate, average" << std::endl;
+ std::cout << "\t\t\tthe default is 'copy'" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * 'copy' is useless with '-s none'" << std::endl;
+ std::cout << "\t\t\t * the difference between 'fadecopy' and" << std::endl;
+ std::cout << "\t\t\t 'fadeaccumulate is cleared when using '-B'" << std::endl;
+ std::cout << "\t-r\t\treverse the trail drawing sequence" << std::endl;
+ std::cout << "\t-B\t\tshow the background behind the trail" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * only used with '-s background'" << std::endl;
+ std::cout << "\t-F\t\tredraw the current frame on top of the trail" << std::endl;
+ std::cout << "\t\t\tNOTES:" << std::endl;
+ std::cout << "\t\t\t * noticeable with '-s average'" << std::endl;
+ std::cout << "\t\t\t * noticeable with reverse faded trails" << std::endl;
+}
+
+int main(int argc, char *argv[])
+{
+ int ret = 0;
+ int opt;
+
+ std::string *input_file = NULL;
+ std::string *output_file = NULL;
+ int trail_lenght = 25;
+ std::string *segmentation_method = new std::string("background");
+ int background_learn_frames = 50;
+ int threshold_level = 5;
+ std::string *drawing_method = new std::string("copy");
+ bool reverse_trail = false;
+ bool show_background = false;
+ bool redraw_current_frame = false;
+
+ while ((opt = getopt(argc, argv, "i:o:l:s:b:t:d:rBFh")) != -1) {
+ switch (opt) {
+ case 'i':
+ input_file = new std::string(optarg);
+ break;
+ case 'o':
+ output_file = new std::string(optarg);
+ break;
+ case 'l':
+ trail_lenght = atoi(optarg);
+ break;
+ case 's':
+ delete segmentation_method;
+ segmentation_method = new std::string(optarg);
+ break;
+ case 'b':
+ background_learn_frames = atoi(optarg);
+ break;
+ case 't':
+ threshold_level = atoi(optarg);
+ break;
+ case 'd':
+ delete drawing_method;
+ drawing_method = new std::string(optarg);
+ break;
+ case 'r':
+ reverse_trail = true;
+ break;
+ case 'B':
+ show_background = true;
+ break;
+ case 'F':
+ redraw_current_frame = true;
+ break;
+ case 'h':
+ usage(argv[0]);
+ return 0;
+ default: /* '?' */
+ usage(argv[0]);
+ return -1;
+ }
+ }
+
+ cv::VideoCapture inputVideo;
+ cv::VideoWriter outputVideo;
+ cv::Size frame_size;
+ cv::Mat input_frame;
+
+ if (input_file) {
+ inputVideo.open(*input_file);
+ } else {
+ inputVideo.open(0);
+ }
+
+ if (!inputVideo.isOpened()) {
+ std::cerr << "Could not open the input video." << std::endl;
+ ret = -1;
+ goto out;
+ }
+
+ frame_size = cv::Size((int) inputVideo.get(CV_CAP_PROP_FRAME_WIDTH),
+ (int) inputVideo.get(CV_CAP_PROP_FRAME_HEIGHT));
+
+ if (output_file) {
+ int fps = inputVideo.get(CV_CAP_PROP_FPS);
+ if (fps < 0)
+ fps = 25;
+
+ outputVideo.open(*output_file, CV_FOURCC('M','J','P','G'), fps, frame_size, true);
+ if (!outputVideo.isOpened()) {
+ std::cerr << "Could not open the output video for write." << std::endl;
+ ret = -1;
+ goto out;
+ }
+ }
+
+ Trail *trail;
+ if (reverse_trail)
+ trail = new ReverseTrail(trail_lenght, frame_size);
+ else
+ trail = new ForwardTrail(trail_lenght, frame_size);
+
+ trail->setRedrawCurrentFrame(redraw_current_frame);
+
+ if (trail->setDrawingMethod(*drawing_method) < 0) {
+ std::cerr << "Invalid drawing method." << std::endl;
+ ret = -1;
+ goto out_delete_trail;
+ }
+
+ Segmentation *segmentation;
+ if (*segmentation_method == "background") {
+ segmentation = new MOG2Segmentation(inputVideo, background_learn_frames);
+ if (show_background) {
+ cv::Mat background;
+
+ ((MOG2Segmentation *)segmentation)->getBackgroundImage(background);
+ trail->setBackground(background);
+ }
+ } else if (*segmentation_method == "threshold") {
+ segmentation = new ThresholdSegmentation(threshold_level);
+ } else if (*segmentation_method == "none") {
+ segmentation = new DummySegmentation();
+ } else {
+ std::cerr << "Invalid segmentation method." << std::endl;
+ ret = -1;
+ goto out_delete_trail;
+ }
+
+ cv::namedWindow("Frame", CV_WINDOW_AUTOSIZE);
+
+ for (;;) {
+ inputVideo >> input_frame;
+
+ Frame *foreground = new Frame(input_frame,
+ segmentation->getForegroundMask(input_frame));
+ trail->update(foreground);
+
+ cv::Mat canvas = cv::Mat::zeros(input_frame.size(), input_frame.type());
+ trail->draw(canvas);
+
+ cv::imshow("Frame", canvas);
+ if (cv::waitKey(30) >= 0)
+ break;
+
+ if (outputVideo.isOpened())
+ outputVideo << canvas;
+ }
+
+ cv::destroyWindow("Frame");
+
+ delete segmentation;
+
+out_delete_trail:
+ delete trail;
+out:
+ delete drawing_method;
+ delete segmentation_method;
+ delete output_file;
+ delete input_file;
+ return ret;
+}
projects / experiments / opencv_trail_effect.git / commitdiff