From 81266999b597eeac96b846034444f4b240da72bb Mon Sep 17 00:00:00 2001 From: Antonio Ospite Date: Fri, 25 Nov 2016 17:11:27 +0100 Subject: [PATCH] Initial import --- .gitignore | 1 + Makefile | 4 + README.md | 55 ++++++++++++++ TODO | 3 + create_test_videofont.py | 70 ++++++++++++++++++ examples/Beyer Op. 101 - Exercise 003.ly | 13 ++++ examples/Beyer Op. 101 - Exercise 003.midi | Bin 0 -> 194 bytes examples/Beyer Op. 101 - Exercise 008.ly | 30 ++++++++ examples/Beyer Op. 101 - Exercise 008.midi | Bin 0 -> 671 bytes examples/ges-simple-player.py | 62 ++++++++++++++++ vidi-player.py | 113 +++++++++++++++++++++++++++++ vidi/Note.py | 92 +++++++++++++++++++++++ vidi/Player.py | 61 ++++++++++++++++ vidi/Timeline.py | 59 +++++++++++++++ vidi/__init__.py | 5 ++ 15 files changed, 568 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 TODO create mode 100755 create_test_videofont.py create mode 100644 examples/Beyer Op. 101 - Exercise 003.ly create mode 100644 examples/Beyer Op. 101 - Exercise 003.midi create mode 100644 examples/Beyer Op. 101 - Exercise 008.ly create mode 100644 examples/Beyer Op. 101 - Exercise 008.midi create mode 100755 examples/ges-simple-player.py create mode 100755 vidi-player.py create mode 100755 vidi/Note.py create mode 100755 vidi/Player.py create mode 100755 vidi/Timeline.py create mode 100755 vidi/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4e58176 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +all: + +clean: + rm -rf ~* *.swp videofont vidi/__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..cf3995e --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +vidi-player creates a video timeline starting from a MIDI file. + +The video clips are taken from a "VideoFont" in which each sample clip +corresponds to a note. The samples are arranged following the time and value of +the notes in the MIDI file. + + +Examples of use +=============== + +Create a synthetinc VideoFont: + + $ ./create_test_videofont.py videofont + + +Play the timeline from a MIDI file using the samples from the VideoFont: + + $ ./vidi-player.py examples/Beyer\ Op.\ 101\ -\ Exercise\ 008.midi videofont + + +Save the timeline to be edited somewhere else (e.g. in PiTiVi): + + $ ./vidi-player.py examples/Beyer\ Op.\ 101\ -\ Exercise\ 008.midi videofont/ Beyer_008.xges + + +Render the timeline to a video file: + + $ ges-launch-1.0 --load Beyer_008.xges --outputuri Beyer_008.webm --format="video/webm:video/x-vp8:audio/x-vorbis" + + +Similar projects +================ + +Of course if the idea is cool, chances are that others had it too. :) + +Midi-Vidi +--------- + +A similar concept called midi-vidi has been developed by Marcus Fischer: + +* http://unrecnow.com/dust/325-midi-vidi-effect-demo/ +* https://vimeo.com/8049917 +* http://www.synthtopia.com/content/2009/12/09/midi-vidi-plugin-lets-you-control-video-via-midi-in-max-for-live/ +* http://www.maxforlive.com/library/device.php?id=120 + +Midi-Vidi is also a very cool name, it sound like the Italian "mi dividi" which +means "you divide me". + +VidiBox +------- + +An interactive video mashup app: + +* http://www.vidibox.net/ +* https://twitter.com/vidibox diff --git a/TODO b/TODO new file mode 100644 index 0000000..b80ca0c --- /dev/null +++ b/TODO @@ -0,0 +1,3 @@ +- Get more stats from the midi file (number of channels, note range, etc.) + The note range in particular can help when creating a VideoFont for + a specific midi file. diff --git a/create_test_videofont.py b/create_test_videofont.py new file mode 100755 index 0000000..d113875 --- /dev/null +++ b/create_test_videofont.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import sys +import os +import vidi + + +LIVE_PIPELINE_TEMPLATE = """ + videotestsrc num-buffers=1 pattern=black ! \ + textoverlay valignment=center halignment=center font-desc="Sans, 72" text="{0}" ! \ + autovideosink \ + audiotestsrc num-buffers=100 freq={1:f} ! \ + autoaudiosink +""" + +#FONT_DESC = "Andale Mono, 72" +FONT_DESC = "Mono, 72" + +FILE_PIPELINE_TEMPLATE = """ + matroskamux name=mux ! filesink location="{2}/sample_{0}.mkv" + videotestsrc num-buffers=1 pattern=black ! \ + textoverlay valignment=center halignment=center font-desc="%s" text="{0}" ! \ + queue ! schroenc rate-control=3 ! mux. + audiotestsrc num-buffers=100 freq={1:f} ! \ + queue ! audioconvert ! vorbisenc quality=0.5 ! queue ! mux. +""" % FONT_DESC + + +def create_test_videofont(pipeline_template, notes_range, destination_dir=None): + for i, note_number in enumerate(notes_range): + note = vidi.SpnNote(note_number) + + print("%2d %s" % (i, note)) + + pipeline_string = pipeline_template.format(note.name, note.frequency, destination_dir) + + player = vidi.Player.from_pipeline_string(pipeline_string) + error = player.play() + if error: + break + + +def usage(): + print("usage: %s []" % + os.path.basename(sys.argv[0])) + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] in ["-h", "--help"]: + usage() + return 0 + + notes_range = vidi.PIANO_88_KEYS_RANGE + + if len(sys.argv) > 1: + destination_dir = os.path.realpath(sys.argv[1]) + if os.path.exists(destination_dir): + sys.stderr.write("The destination already exists '%s'!\n" + % destination_dir) + return 1 + + os.mkdir(destination_dir) + create_test_videofont(FILE_PIPELINE_TEMPLATE, notes_range, + destination_dir) + else: + create_test_videofont(LIVE_PIPELINE_TEMPLATE, notes_range) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/Beyer Op. 101 - Exercise 003.ly b/examples/Beyer Op. 101 - Exercise 003.ly new file mode 100644 index 0000000..866ea57 --- /dev/null +++ b/examples/Beyer Op. 101 - Exercise 003.ly @@ -0,0 +1,13 @@ +\version "2.19.51" + +\score { + \relative c'' { + \time 4/4 + \tempo 4 = 120 + c1 d e f + g f e d + c + } + \layout { } + \midi { } +} diff --git a/examples/Beyer Op. 101 - Exercise 003.midi b/examples/Beyer Op. 101 - Exercise 003.midi new file mode 100644 index 0000000000000000000000000000000000000000..5dd676df9defd7340ce7d3bbff99ac5482f94686 GIT binary patch literal 194 zcmeYb$w*;fU|?flWME=!@C_--2J+k({xdT${Ac7$E=o--$uF`}0CMEq{X!LdGIJ^e z^7B#@jPwjG^-K*F;DF(O1Pcq35C_BmKxX!ZKn?l~P}Aaorm@E4rIuSUOz?>60TK)h k6TCnSFCfDQ#P9(!d_fFfAj2QT@JAMbtAXo+nFF*10E$5~R{#J2 literal 0 HcmV?d00001 diff --git a/examples/Beyer Op. 101 - Exercise 008.ly b/examples/Beyer Op. 101 - Exercise 008.ly new file mode 100644 index 0000000..ec41f96 --- /dev/null +++ b/examples/Beyer Op. 101 - Exercise 008.ly @@ -0,0 +1,30 @@ +\version "2.19.51" + +\score { + \relative c'' { + \time 4/4 + \tempo 4 = 120 + c4 ( -1 e4 -3 c4 -1 e4 -3 + g4 -5 c,4 ) c4 c4 ( + d4 ) d4 d4 d4 ( + e4 ) e4 e4 e4 ( + c4 e4 c4 e4 + g4 c,4 ) c4 c4 ( + d4 ) d4 ( e4 d4 + c4 e4 c2 ) \bar "||" + \break \repeat volta 2 { + + g'4 ( -5 d4 ) -2 d4 d4 ( + | \barNumberCheck #10 + e4 -3 c4 ) -1 c4 c4 ( + g'4 d4 ) d4 d4 ( + e4 c4 e4 d4 + c4 ) ( e4 c4 e4 + g4 c,4 ) c4 c4 ( + d4 ) d4 ( e4 d4 + c4 e4 c2 ) + } + } + \layout { } + \midi { } +} diff --git a/examples/Beyer Op. 101 - Exercise 008.midi b/examples/Beyer Op. 101 - Exercise 008.midi new file mode 100644 index 0000000000000000000000000000000000000000..10e8a3042af21dda5ec63a5328bfaea0be3a2817 GIT binary patch literal 671 zcmeYb$w*;fU|?flWME=!@C_--2J+k({xdT${Ac7$E=o--$uF`}0CMEq{X!LdGIJ^e z^7B#@jPwjG^-K*F;DF(O1Pcq35C_BmKxX!ZKn?l~5Yw2ffu^y>kt-!;88wgi}DMFrm;0D57h-tD%R2w8T z;8wsDV=~|d!p*^CkY^6uWVoepli^aBB5=FlQkV?5fp7-gK)4887hDP!EWjiL09XdP A^Z)<= literal 0 HcmV?d00001 diff --git a/examples/ges-simple-player.py b/examples/ges-simple-player.py new file mode 100755 index 0000000..bfcf330 --- /dev/null +++ b/examples/ges-simple-player.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# ges-simple-player - simple GStreamer Editing Service example +# +# Copyright (C) 2016 Antonio Ospite +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import gi + +# XXX it's an ugly hack, I know +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +import vidi + +gi.require_version('Gst', '1.0') +from gi.repository import Gst +Gst.init(None) + +gi.require_version('GES', '1.0') +from gi.repository import GES +GES.init() + +def play_clip(video_path): + + timeline = GES.Timeline.new_audio_video() + layer = timeline.append_layer() + + video_uri = "file://" + video_path + asset = GES.UriClipAsset.request_sync(video_uri) + clip = layer.add_asset(asset, 0, 0, asset.get_duration(), GES.TrackType.UNKNOWN) + + timeline.commit() + + pipeline = GES.Pipeline() + pipeline.set_timeline(timeline) + + vidi.Player(pipeline).play() + + +def main(): + if len(sys.argv) < 2: + return 1 + + video_path = os.path.realpath(sys.argv[1]) + play_clip(video_path) + + +if __name__ == "__main__": + sys,exit(main()) diff --git a/vidi-player.py b/vidi-player.py new file mode 100755 index 0000000..dbd5a75 --- /dev/null +++ b/vidi-player.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# +# vidi-player - generate GStreamer Editing Services timelines from midi +# +# Copyright (C) 2016 Antonio Ospite +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import mido +import vidi + + +def is_note(msg): + return msg.type == 'note_on' or msg.type == 'note_off' + + +def is_note_on(msg): + return msg.type == 'note_on' and msg.velocity > 0 + + +def is_note_off(msg): + return ((msg.type == 'note_on' and msg.velocity == 0) or + (msg.type == 'note_off')) + + +def check_overlapping_notes(midi_file): + previous_note_on = False + for msg in midi_file: + if is_note_on(msg) and msg.channel == 0: + if previous_note_on: + return True + + previous_note_on = True + elif is_note_off(msg) and msg.channel == 0: + previous_note_on = False + + return False + + +def timeline_from_midi(midi_file, video_font_path): + timeline = vidi.Timeline() + + elapsed_time = 0 + start_time = 0 + for msg in midi_file: + elapsed_time += msg.time + if is_note_on(msg) and msg.channel == 0: + start_time = elapsed_time + elif is_note_off(msg) and msg.channel == 0: + note = vidi.MidiNote(msg.note) + duration = elapsed_time - start_time + print("Note name: %s start_time: %f duration: %f" % + (note.name, start_time, duration)) + + video_sample_uri = "file://%s/sample_%s.mkv" % (video_font_path, note.name) + + timeline.add_clip(video_sample_uri, start_time, duration) + + return timeline + + +def usage(): + print("usage: %s []" + % os.path.basename(sys.argv[0])) + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] in ["-h", "--help"]: + usage() + return 0 + + if len(sys.argv) < 3: + usage() + return 1 + + midi_file = mido.MidiFile(sys.argv[1]) + + overlapping_notes = check_overlapping_notes(midi_file) + if overlapping_notes: + sys.stderr.write("Sorry, supporting only midi file with no overlapping notes") + return 1 + + if not os.path.isdir(sys.argv[2]): + sys.stderr.write("The second argument must be the path of the videofont directory") + usage() + return 1 + + video_font_path = os.path.realpath(sys.argv[2]) + + timeline = timeline_from_midi(midi_file, video_font_path) + + if len(sys.argv) > 3: + destination_uri = 'file://' + os.path.realpath(sys.argv[3]) + timeline.save(destination_uri) + else: + timeline.play() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vidi/Note.py b/vidi/Note.py new file mode 100755 index 0000000..86c6796 --- /dev/null +++ b/vidi/Note.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Music - an utility class to deal with musical details +# +# Copyright (C) 2016 Antonio Ospite +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# The 88 piano keyboard goes from A0 (-39) to C8 (+48), see: +# https://en.wikipedia.org/wiki/Piano_key_frequencies +PIANO_88_KEYS_RANGE = range(-39, 48 + 1) + + +class SpnNote(object): + def __init__(self, note_number): + # Scientific Pitch Notation values are from -48 to +83 + # https://en.wikipedia.org/wiki/Scientific_pitch_notation + if note_number < -48 or note_number > 83: + raise ValueError("Invalid Scientific Pitch Notation number") + + self.note_number = note_number + self.name = self._name() + self.frequency = self._frequency() + + def _name(self): + note_names = ["C", "C#", "D", "Eb", "E", "F", "F#", "G", "Ab", "A", "Bb", "B"] + + note_offset = self.note_number % 12 + name = note_names[note_offset] + octave = (self.note_number + 48 - note_offset) // 12 + note_name = "%s%d" % (name, octave) + return note_name + + def _frequency(self): + # https://en.wikipedia.org/wiki/A440_(pitch_standard) + return round(440 * pow(2, (self.note_number - 9) / 12), 4) + + def __repr__(self): + return "(%+3d): %-3s frequency: %9.4f" % (self.note_number, + self.name, + self.frequency) + + +class MidiNote(SpnNote): + def __init__(self, note_number): + # midi notes go from 0 to 127 + if note_number < 0 or note_number > 127: + raise ValueError("Invalid midi note") + + # In Scientific Pitch Notation C4 is 0 + # In MIDI it's 60 + SpnNote.__init__(self, note_number - 60) + + +def test(): + A0 = SpnNote(-39) + print("Note A0? %s" % A0) + del A0 + + C4 = SpnNote(0) + print("Note C4? %s" % C4) + del C4 + + A4 = SpnNote(9) + print("Note A4? %s" % A4) + del A4 + + C8 = SpnNote(+48) + print("Note C8? %s" % C8) + del C8 + + print() + + midi_C4 = MidiNote(60) + print("Midi C4? %s" % midi_C4) + del midi_C4 + + + +if __name__ == "__main__": + test() diff --git a/vidi/Player.py b/vidi/Player.py new file mode 100755 index 0000000..26a9832 --- /dev/null +++ b/vidi/Player.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# +# Player - a very simple GStreamer player +# +# Copyright (C) 2016 Antonio Ospite +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import gi + +gi.require_version('Gst', '1.0') +from gi.repository import Gst +Gst.init(None) + +from gi.repository import GObject +GObject.threads_init() + + +class Player(object): + def __init__(self, pipeline): + self.pipeline = pipeline + self.mainloop = GObject.MainLoop() + + @staticmethod + def from_pipeline_string(pipeline_string): + pipeline = Gst.parse_launch(pipeline_string) + return Player(pipeline) + + def quit(self): + self.mainloop.quit() + self.pipeline.set_state(Gst.State.NULL) + + def bus_message_cb(self, unused_bus, message): + if message.type == Gst.MessageType.EOS: + self.quit() + + def play(self): + bus = self.pipeline.get_bus() + bus.add_signal_watch() + bus.connect("message", self.bus_message_cb) + + self.pipeline.set_state(Gst.State.PLAYING) + + try: + self.mainloop.run() + except KeyboardInterrupt: + self.quit() + return 1 + + return 0 diff --git a/vidi/Timeline.py b/vidi/Timeline.py new file mode 100755 index 0000000..48c6181 --- /dev/null +++ b/vidi/Timeline.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# +# Timeline - very simple GES timeline wrapper +# +# Copyright (C) 2016 Antonio Ospite +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import gi + +gi.require_version('Gst', '1.0') +from gi.repository import Gst +Gst.init(None) + +gi.require_version('GES', '1.0') +from gi.repository import GES +GES.init() + +from .Player import Player + + +class Timeline(object): + def __init__(self): + self.project = GES.Project(extractable_type=GES.Timeline) + self.timeline = GES.Asset.extract(self.project) + + audio_track = GES.AudioTrack.new() + video_track = GES.VideoTrack.new() + + self.timeline.add_track(audio_track) + self.timeline.add_track(video_track) + + self.layer = self.timeline.append_layer() + + def add_clip(self, clip_uri, start_time, duration): + asset = GES.UriClipAsset.request_sync(clip_uri) + self.layer.add_asset(asset, start_time * Gst.SECOND, 0, + duration * Gst.SECOND, GES.TrackType.UNKNOWN) + + def play(self): + self.timeline.commit() + + ges_pipeline = GES.Pipeline() + ges_pipeline.set_timeline(self.timeline) + Player(ges_pipeline).play() + + def save(self, uri): + self.project.save(self.timeline, uri, None, False) diff --git a/vidi/__init__.py b/vidi/__init__.py new file mode 100755 index 0000000..060bc66 --- /dev/null +++ b/vidi/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from .Note import SpnNote, MidiNote, PIANO_88_KEYS_RANGE +from .Player import Player +from .Timeline import Timeline -- 2.1.4