From cec3fd77811fe36a6f5ef90ff06dd9b42979866c Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 12 Oct 2021 13:41:50 +0200 Subject: [PATCH 01/59] Added download of slides, Changed _download to use a session object to perform requests via --- daemon/packager.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 2d4a2c8..ff26b31 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -6,6 +6,7 @@ import shutil import requests from requests.auth import HTTPBasicAuth +from requests.sessions import Session from daisy import DaisyHandler @@ -46,23 +47,29 @@ class Mediasite: if 'id' in data: mypackage['notification_id'] = data['id'] - mypackage['thumb'] = self._download(base, data['thumb'], self.auth) - for source in data['sources']: - mysource = {'video': self._download(base, source['video']), - 'poster': self._download(base, source['poster'], - self.auth), - 'playAudio': source['playAudio']} - mypackage['sources'].append(mysource) + with requests.Session() as session: + session.auth = self.auth + session.stream = True + mypackage['thumb'] = self._download(base, data['thumb'], session) + for source in data['sources']: + mysource = {'video': self._download(base, source['video'], session), + 'poster': self._download(base, source['poster'], session), + 'playAudio': source['playAudio']} + mypackage['sources'].append(mysource) + + mypackage['slides'] = [ + setattr(elem, 'url', os.path.join(base, self._download(base, elem['url'], session))) for elem in data['slides'] + ] return mypackage - def _download(self, base, remotefile, auth=None): + def _download(self, base, remotefile, session: Session): localname = remotefile.split('/')[-1] localpath = os.path.join(base, localname) - with requests.get(remotefile, stream=True, auth=auth) as r: - r.raise_for_status() - with open(localpath, 'xb') as f: - for chunk in r.iter_content(chunk_size=self.chunk_size): - f.write(chunk) + r = session.get(remotefile) + r.raise_for_status() + with open(localpath, 'xb') as f: + for chunk in r.iter_content(chunk_size=self.chunk_size): + f.write(chunk) return localname -- 2.39.5 From aebd5a2d375351528431877c68910b3cc00c8bf5 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 12 Oct 2021 20:21:12 +0200 Subject: [PATCH 02/59] Changed the creation of a slidfes property since to accomodate for python2.7 --- daemon/packager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index ff26b31..2eaa939 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -57,9 +57,15 @@ class Mediasite: 'playAudio': source['playAudio']} mypackage['sources'].append(mysource) - mypackage['slides'] = [ - setattr(elem, 'url', os.path.join(base, self._download(base, elem['url'], session))) for elem in data['slides'] - ] + if 'slides' in data: + slides_path = os.join.path(base, 'slides') + os.mkdir(slides_path) + mypackage['slides'] = [] + for slide in data['slides']: + myslide = { 'url': os.join.path(slides_path, self._download(slides_path, slide['url'], session)), + 'duration': slide['duration'] } + mypackage['slides'].append(myslide) + return mypackage def _download(self, base, remotefile, session: Session): -- 2.39.5 From 1e6f8608952929016ea974c1c180a65d2cc96f90 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 13 Oct 2021 09:31:09 +0200 Subject: [PATCH 03/59] Changed function _download parameters to not have static typing --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 2eaa939..4b0e289 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -68,7 +68,7 @@ class Mediasite: return mypackage - def _download(self, base, remotefile, session: Session): + def _download(self, base, remotefile, session): localname = remotefile.split('/')[-1] localpath = os.path.join(base, localname) r = session.get(remotefile) -- 2.39.5 From 1cdf3e6f67b7e72385e151049408879dcd84d608 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 13 Oct 2021 13:17:00 +0200 Subject: [PATCH 04/59] Started work on make_slides_video function --- daemon/packager.py | 18 +++++++++++++----- daemon/transcoder.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 4b0e289..510ef6c 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -61,11 +61,19 @@ class Mediasite: slides_path = os.join.path(base, 'slides') os.mkdir(slides_path) mypackage['slides'] = [] - for slide in data['slides']: - myslide = { 'url': os.join.path(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': slide['duration'] } - mypackage['slides'].append(myslide) - + demux_file = os.join.path(slides_path, 'demux.txt') + with open(demux_file, 'w') as f: + for slide in data['slides']: + myslide = { 'url': os.join.path(slides_path, self._download(slides_path, slide['url'], session)), + 'duration': slide['duration'] } + f.write('file \'{}\'\n'.format(myslide['url'])) + f.write('duration {}\n'.format(slide['duration'])) + mypackage['slides'].append(myslide) + + # to accomodate for an ffmpeg quirk that needs the last slide twice + f.write('file \'{}\'\n'.format(mypackage['slides'][-1]['url'])) + mypackage['demux_file'] = demux_file + return mypackage def _download(self, base, remotefile, session): diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 5defed9..7c7619a 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -1,6 +1,6 @@ import logging import multiprocessing as mp -import os +import oset = True import shutil import time @@ -66,6 +66,15 @@ class Transcoder: pending.append({'jobs': transcodes, 'data': stream}) + + if 'slides' in package: + slides_job = pool.apply_async(_Worker.make_slides_video, + (self.worker, + package['demux_file'], + workbase, + maxheight, preset, crf)) + + pool.close() pool.join() self.logger.info("%s - Finished transcoding", package_id) @@ -148,3 +157,21 @@ class _Worker: self.logger.info("%s - Transcoded %s in %s seconds", package_id, outstream, runtime) return outstream + + def make_slides_video(self, package_id, demux_file, outdir, maxheight, preset, crf): + self.logger.debug("%s - Preparing slides for processing", package_id) + outstream = '{}-{}-{}-{}.mp4'.format('slides', maxheight, preset, crf) + outpath = os.path.join(outdir, outstream) + self.logger.debug("%s - Building slide video from demux.txt", package_id) + + start = time.time() + quiet = True + if self.logger.getEffectiveLevel() < logging.INFO: + quiet = False + + result, _ = (ffmpeg + .input(demux_file, framerate=25) + .filter('scale', height='min(in_h, {}').format(maxheight, width=-2) + .output(outpath, crf=crf, preset=preset, movflags='faststart', pix_fmt='yuv420p') + .run(quiet=quiet)) + \ No newline at end of file -- 2.39.5 From 7a29a3b3b32955fbf42cff3313bf40dacc9462be Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 13 Oct 2021 13:26:49 +0200 Subject: [PATCH 05/59] Fixes --- daemon/transcoder.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 7c7619a..d3697e6 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -1,8 +1,9 @@ import logging import multiprocessing as mp -import oset = True +import os import shutil import time +from typing_extensions import runtime import ffmpeg @@ -164,14 +165,18 @@ class _Worker: outpath = os.path.join(outdir, outstream) self.logger.debug("%s - Building slide video from demux.txt", package_id) - start = time.time() quiet = True if self.logger.getEffectiveLevel() < logging.INFO: quiet = False - result, _ = (ffmpeg - .input(demux_file, framerate=25) - .filter('scale', height='min(in_h, {}').format(maxheight, width=-2) - .output(outpath, crf=crf, preset=preset, movflags='faststart', pix_fmt='yuv420p') - .run(quiet=quiet)) - \ No newline at end of file + start = time.time() + (ffmpeg + .input(demux_file, framerate=25) + .filter('scale', height='min(in_h, {}').format(maxheight, width=-2) + .output(outpath, crf=crf, preset=preset, movflags='faststart', pix_fmt='yuv420p') + .run(quiet=quiet)) + + runtime = time.time() - start + self.logger.info("%s - Transcoded %s in %s seconds", package_id, outstream, runtime) + + return outstream \ No newline at end of file -- 2.39.5 From 3dc8db5cd9cbe6c805960060aad5006035636ce7 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 13 Oct 2021 13:29:08 +0200 Subject: [PATCH 06/59] Added loggin --- daemon/transcoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index d3697e6..7120755 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -169,6 +169,7 @@ class _Worker: if self.logger.getEffectiveLevel() < logging.INFO: quiet = False + self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg .input(demux_file, framerate=25) -- 2.39.5 From ea6a6ef7aa379510649bc3c91bdfcbfe88977374 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 13 Oct 2021 17:06:48 +0200 Subject: [PATCH 07/59] Added most of the jobs for the transcoder. ready for testing --- daemon/transcoder.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 7120755..c314196 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -65,16 +65,20 @@ class Transcoder: workbase)) transcodes['poster'] = thumbjob - pending.append({'jobs': transcodes, - 'data': stream}) - + obj = {'jobs': transcodes, + 'data': stream} if 'slides' in package: - slides_job = pool.apply_async(_Worker.make_slides_video, - (self.worker, - package['demux_file'], - workbase, - maxheight, preset, crf)) + slides = {} + for maxheight in self.variants: + crf, preset = self.variants[maxheight] + slides[maxheight] = pool.apply_async(_Worker.make_slides_video, + (self.worker, + package['demux_file'], + workbase, + maxheight, preset, crf)) + obj['slides'] = slides + pending.append(obj) pool.close() pool.join() @@ -84,10 +88,16 @@ class Transcoder: for item in pending: stream = item['data'] jobs = item['jobs'] + slide_jobs = item.get('slides') streams = {} + slides = {} for maxheight in self.variants: streams[maxheight] = jobs[maxheight].get() + if slide_jobs is not None: + slides[maxheight] = slide_jobs[maxheight].get() stream['video'] = streams + if slide_jobs is not None: + stream['slides'] = slides if 'poster' in jobs: stream['poster'] = jobs['poster'].get() if not package['thumb'] and stream['playAudio']: @@ -174,7 +184,7 @@ class _Worker: (ffmpeg .input(demux_file, framerate=25) .filter('scale', height='min(in_h, {}').format(maxheight, width=-2) - .output(outpath, crf=crf, preset=preset, movflags='faststart', pix_fmt='yuv420p') + .output(outpath, crf=crf, preset=preset, vsync='vfr', movflags='faststart', pix_fmt='yuv420p') .run(quiet=quiet)) runtime = time.time() - start -- 2.39.5 From 13f6ef1b32c4378ffe247c085f36d2f7de12e66b Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 10:36:02 +0200 Subject: [PATCH 08/59] removed typing_extension --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index c314196..1a02fca 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -3,7 +3,7 @@ import multiprocessing as mp import os import shutil import time -from typing_extensions import runtime +#from typing_extensions import runtime import ffmpeg -- 2.39.5 From e5f61b30aa9e36f760fba20751d0dddb8578d4b6 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 10:41:25 +0200 Subject: [PATCH 09/59] fixed os.path.join error --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 510ef6c..17954b9 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -58,10 +58,10 @@ class Mediasite: mypackage['sources'].append(mysource) if 'slides' in data: - slides_path = os.join.path(base, 'slides') + slides_path = os.path.join(base, 'slides') os.mkdir(slides_path) mypackage['slides'] = [] - demux_file = os.join.path(slides_path, 'demux.txt') + demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.join.path(slides_path, self._download(slides_path, slide['url'], session)), -- 2.39.5 From c80be26c2c896b45a62d0c68f8a1ee3eeeb3f30d Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 10:43:53 +0200 Subject: [PATCH 10/59] another os fix --- daemon/packager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 17954b9..63ea078 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -64,7 +64,7 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: for slide in data['slides']: - myslide = { 'url': os.join.path(slides_path, self._download(slides_path, slide['url'], session)), + myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), 'duration': slide['duration'] } f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(slide['duration'])) @@ -73,7 +73,6 @@ class Mediasite: # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(mypackage['slides'][-1]['url'])) mypackage['demux_file'] = demux_file - return mypackage def _download(self, base, remotefile, session): -- 2.39.5 From f42ac62131adc7ab6ed292dc8d50215e18098a2a Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 10:45:53 +0200 Subject: [PATCH 11/59] fixed keyerror. maybe.... --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 63ea078..1fd6976 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -65,9 +65,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': slide['duration'] } + 'duration': slide['time'] } f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(slide['duration'])) + f.write('duration {}\n'.format(slide['time'])) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice -- 2.39.5 From 73b6cb119eee60b7152e9728447c24bd74d54960 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 10:57:11 +0200 Subject: [PATCH 12/59] fixed missed postionala rgument --- daemon/transcoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 1a02fca..ecab1ee 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -73,6 +73,7 @@ class Transcoder: crf, preset = self.variants[maxheight] slides[maxheight] = pool.apply_async(_Worker.make_slides_video, (self.worker, + package_id, package['demux_file'], workbase, maxheight, preset, crf)) -- 2.39.5 From 79aa76f106613f6f0cc6f8c8ceda31ff0e427a63 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:03:28 +0200 Subject: [PATCH 13/59] added logging --- daemon/packager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 1fd6976..dc80de1 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -3,6 +3,7 @@ import json import os import re import shutil +import logging import requests from requests.auth import HTTPBasicAuth @@ -23,6 +24,7 @@ class Mediasite: config['mediasite']['password']) self.chunk_size = 10485760 # 10MiB; seems optimal for speed # Tested 8k, 10MiB and 20MiB + self.logger = logging.getLogger('play-daemon') def pack(self, pres_id, queue_item): data = queue_item['data'] @@ -64,10 +66,11 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: for slide in data['slides']: + self.logger.info('Slide: {}'.format(slide)) myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': slide['time'] } + 'duration': slide['duration'] } f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(slide['time'])) + f.write('duration {}\n'.format(slide['duration'])) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice -- 2.39.5 From c9051f9e9bbb06a48789d531a442bba2f2018a9d Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:14:34 +0200 Subject: [PATCH 14/59] fixed parenthesis error --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index ecab1ee..32c3151 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -184,7 +184,7 @@ class _Worker: start = time.time() (ffmpeg .input(demux_file, framerate=25) - .filter('scale', height='min(in_h, {}').format(maxheight, width=-2) + .filter('scale', height='min(in_h, {}'.format(maxheight), width=-2) .output(outpath, crf=crf, preset=preset, vsync='vfr', movflags='faststart', pix_fmt='yuv420p') .run(quiet=quiet)) -- 2.39.5 From b04ee56cb977d9c72c6ac752b0dd29c014ca3799 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:34:01 +0200 Subject: [PATCH 15/59] making ffmpeg not quiet --- daemon/transcoder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 32c3151..d31744d 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,16 +176,16 @@ class _Worker: outpath = os.path.join(outdir, outstream) self.logger.debug("%s - Building slide video from demux.txt", package_id) - quiet = True - if self.logger.getEffectiveLevel() < logging.INFO: - quiet = False + quiet = False +# if self.logger.getEffectiveLevel() < logging.INFO: +# quiet = False self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg .input(demux_file, framerate=25) .filter('scale', height='min(in_h, {}'.format(maxheight), width=-2) - .output(outpath, crf=crf, preset=preset, vsync='vfr', movflags='faststart', pix_fmt='yuv420p') + .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') .run(quiet=quiet)) runtime = time.time() - start -- 2.39.5 From 455e0ead6a94b0d1c8dba45a151bf22f138d55fb Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:54:13 +0200 Subject: [PATCH 16/59] fixed another parenthesis misplacement... --- daemon/packager.py | 2 +- daemon/transcoder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index dc80de1..89b925b 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -66,9 +66,9 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: for slide in data['slides']: - self.logger.info('Slide: {}'.format(slide)) myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), 'duration': slide['duration'] } + self.logger.info('Slide: {}'.format(myslide)) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(slide['duration'])) mypackage['slides'].append(myslide) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index d31744d..8fe1e4b 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -184,7 +184,7 @@ class _Worker: start = time.time() (ffmpeg .input(demux_file, framerate=25) - .filter('scale', height='min(in_h, {}'.format(maxheight), width=-2) + .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') .run(quiet=quiet)) -- 2.39.5 From 9867ce17f9254aa37166a41705a4ab365ad5cd25 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:56:13 +0200 Subject: [PATCH 17/59] ngh --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 89b925b..a9fce7f 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -68,7 +68,7 @@ class Mediasite: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), 'duration': slide['duration'] } - self.logger.info('Slide: {}'.format(myslide)) + self.logger.info('MySlide: {}'.format(myslide)) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(slide['duration'])) mypackage['slides'].append(myslide) -- 2.39.5 From eb18e221d1c69cb24721dbacc34ee8e5daa2fd0e Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 11:59:30 +0200 Subject: [PATCH 18/59] overwrite old slides output --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 8fe1e4b..ace4af3 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -185,7 +185,7 @@ class _Worker: (ffmpeg .input(demux_file, framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) - .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') + .overwrite_output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') .run(quiet=quiet)) runtime = time.time() - start -- 2.39.5 From bdb851e088340c6dc6ed402cb96d961b0ab37337 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 12:07:36 +0200 Subject: [PATCH 19/59] chagned ffmpeg call, removed some loggin --- daemon/packager.py | 1 - daemon/transcoder.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index a9fce7f..f81e424 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -68,7 +68,6 @@ class Mediasite: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), 'duration': slide['duration'] } - self.logger.info('MySlide: {}'.format(myslide)) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(slide['duration'])) mypackage['slides'].append(myslide) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index ace4af3..8211337 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -185,7 +185,8 @@ class _Worker: (ffmpeg .input(demux_file, framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) - .overwrite_output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') + .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') + .overwrite_output() .run(quiet=quiet)) runtime = time.time() - start -- 2.39.5 From 79b154b342dcfb5ab2b2b74768f5610e13817eb5 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 13:23:03 +0200 Subject: [PATCH 20/59] trying other way of inputing to ffmpeg --- daemon/transcoder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 8211337..53a0c6b 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,14 +176,14 @@ class _Worker: outpath = os.path.join(outdir, outstream) self.logger.debug("%s - Building slide video from demux.txt", package_id) - quiet = False -# if self.logger.getEffectiveLevel() < logging.INFO: -# quiet = False + quiet = True + if self.logger.getEffectiveLevel() < logging.INFO: + quiet = False self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, framerate=25) + .input(demux_file, format='concat', framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') .overwrite_output() -- 2.39.5 From 4ff4fa61814f2b659bd06b7a7c66072aba388aac Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 13:28:51 +0200 Subject: [PATCH 21/59] fixed an extra option for ffmpeg concat --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 53a0c6b..b8b9ebe 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -183,7 +183,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', framerate=25) + .input(demux_file, format='concat', safe=0, framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') .overwrite_output() -- 2.39.5 From c68fb977a85d44b0a3fc9e49ef902d89bb5916af Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 13:58:23 +0200 Subject: [PATCH 22/59] sukk --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index b8b9ebe..358b0d5 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -185,7 +185,7 @@ class _Worker: (ffmpeg .input(demux_file, format='concat', safe=0, framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) - .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p') + .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() .run(quiet=quiet)) -- 2.39.5 From 5bc546e4cd4428270ee577e8b9553e772e082cf0 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 14 Oct 2021 14:04:55 +0200 Subject: [PATCH 23/59] removed option --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 358b0d5..ba61ebb 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -183,7 +183,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', safe=0, framerate=25) + .input(demux_file, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From abde9982b27fa80ffa8bfc9bdcc11b06244c0361 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 10:18:35 +0200 Subject: [PATCH 24/59] changed millis -> seconds --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index f81e424..bd75913 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -67,9 +67,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': slide['duration'] } + 'duration': slide['duration'] / 1000} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(slide['duration'])) + f.write('duration {}\n'.format(slide['duration'] / 1000)) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice -- 2.39.5 From 57087b3666e4524350aa41517f37cbbaf4f24a06 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 10:51:19 +0200 Subject: [PATCH 25/59] convert string to int --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index bd75913..013818d 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -67,9 +67,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': slide['duration'] / 1000} + 'duration': int(slide['duration']) / 1000} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(slide['duration'] / 1000)) + f.write('duration {}\n'.format(int(slide['duration']) / 1000)) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice -- 2.39.5 From c68d30c750dd6550258caf91578a30c8a8334a7a Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 11:01:01 +0200 Subject: [PATCH 26/59] testing 1024 --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 013818d..a04a93a 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -67,9 +67,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': int(slide['duration']) / 1000} + 'duration': int(slide['duration']) / 1024} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(int(slide['duration']) / 1000)) + f.write('duration {}\n'.format(int(slide['duration']) / 1024)) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice -- 2.39.5 From 9b4c610ada29e59ef0b43c342c7c5b55b62f1c6e Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 11:17:06 +0200 Subject: [PATCH 27/59] changed to 10 fps --- daemon/packager.py | 4 ++-- daemon/transcoder.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index a04a93a..013818d 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -67,9 +67,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': int(slide['duration']) / 1024} + 'duration': int(slide['duration']) / 1000} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(int(slide['duration']) / 1024)) + f.write('duration {}\n'.format(int(slide['duration']) / 1000)) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice diff --git a/daemon/transcoder.py b/daemon/transcoder.py index ba61ebb..d6b622e 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -185,6 +185,7 @@ class _Worker: (ffmpeg .input(demux_file, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) + .filter('fps', fps=10) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() .run(quiet=quiet)) -- 2.39.5 From 336b378d472f7e4e326720d622e4cd3caddb53cb Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 11:42:41 +0200 Subject: [PATCH 28/59] changed notation from seconds to ms --- daemon/packager.py | 4 ++-- daemon/transcoder.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 013818d..d6104a2 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -67,9 +67,9 @@ class Mediasite: with open(demux_file, 'w') as f: for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': int(slide['duration']) / 1000} + 'duration': '{}ms'.format(slide['duration'])} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}\n'.format(int(slide['duration']) / 1000)) + f.write('duration {}ms\n'.format(slide['duration'])) mypackage['slides'].append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice diff --git a/daemon/transcoder.py b/daemon/transcoder.py index d6b622e..ba61ebb 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -185,7 +185,6 @@ class _Worker: (ffmpeg .input(demux_file, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) - .filter('fps', fps=10) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() .run(quiet=quiet)) -- 2.39.5 From 04f18d26720e5dc6542492945527a163ecb7abe2 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 14:18:29 +0200 Subject: [PATCH 29/59] moved slides -> individual streams as the rest --- daemon/packager.py | 10 +++--- daemon/transcoder.py | 84 ++++++++++++++++++++------------------------ 2 files changed, 44 insertions(+), 50 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index d6104a2..35854e8 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -62,7 +62,7 @@ class Mediasite: if 'slides' in data: slides_path = os.path.join(base, 'slides') os.mkdir(slides_path) - mypackage['slides'] = [] + slides = [] demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: for slide in data['slides']: @@ -70,11 +70,13 @@ class Mediasite: 'duration': '{}ms'.format(slide['duration'])} f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}ms\n'.format(slide['duration'])) - mypackage['slides'].append(myslide) + slides.append(myslide) # to accomodate for an ffmpeg quirk that needs the last slide twice - f.write('file \'{}\'\n'.format(mypackage['slides'][-1]['url'])) - mypackage['demux_file'] = demux_file + f.write('file \'{}\'\n'.format(slides[-1]['url'])) + mypackage['sources'].append({ 'demux_file': demux_file, + 'poster': slides[0]['url'], + 'playAudio': "false" }) return mypackage def _download(self, base, remotefile, session): diff --git a/daemon/transcoder.py b/daemon/transcoder.py index ba61ebb..adb1445 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -3,7 +3,6 @@ import multiprocessing as mp import os import shutil import time -#from typing_extensions import runtime import ffmpeg @@ -34,52 +33,51 @@ class Transcoder: self.logger.debug("%s - Pool created", package_id) for stream in package['sources']: transcodes = {} - streampath_rel = stream['video'] - streampath_abs = os.path.join(base, streampath_rel) - self.logger.debug("%s - Processing stream %s", - package_id, - streampath_rel) - for maxheight in self.variants: - crf, preset = self.variants[maxheight] - videojob = pool.apply_async(_Worker.transcode, - (self.worker, - package_id, - streampath_abs, - workbase, - maxheight, preset, crf)) - transcodes[maxheight] = videojob - - thumbpath_rel = stream['poster'] - if thumbpath_rel: - _, ext = os.path.splitext(thumbpath_rel) - thumbbase, _ = os.path.splitext( - os.path.basename(streampath_abs)) - thumbname = '{}{}'.format(thumbbase, ext) - shutil.copy2(os.path.join(base, thumbpath_rel), - os.path.join(workbase, thumbname)) - stream['poster'] = thumbname - else: - thumbjob = pool.apply_async(_Worker.make_thumb, - (self.worker, - streampath_abs, - workbase)) - transcodes['poster'] = thumbjob - - obj = {'jobs': transcodes, - 'data': stream} - if 'slides' in package: - slides = {} + if 'demux_file' in stream: + self.logger.debug("%s - Processing stream %s", package_id, 'slides job in demux.txt') for maxheight in self.variants: crf, preset = self.variants[maxheight] - slides[maxheight] = pool.apply_async(_Worker.make_slides_video, + transcodes[maxheight] = pool.apply_async(_Worker.make_slides_video, (self.worker, package_id, - package['demux_file'], + stream['demux_file'], workbase, maxheight, preset, crf)) + _, ext = os.path.splitext(stream['poster']) + slides_thumb = 'slides_thumb{}'.format(ext) + shutil.copy2(stream['poster'],os.path.join(workbase, slides_thumb)) + stream['poster'] = slides_thumb - obj['slides'] = slides - pending.append(obj) + elif 'video' in stream: + streampath_rel = stream['video'] + streampath_abs = os.path.join(base, streampath_rel) + self.logger.debug("%s - Processing stream %s", package_id, streampath_rel) + for maxheight in self.variants: + crf, preset = self.variants[maxheight] + transcodes[maxheight] = pool.apply_async(_Worker.transcode, + (self.worker, + package_id, + streampath_abs, + workbase, + maxheight, preset, crf)) + + thumbpath_rel = stream['poster'] + if thumbpath_rel: + _, ext = os.path.splitext(thumbpath_rel) + thumbbase, _ = os.path.splitext(os.path.basename(streampath_abs)) + thumbname = '{}{}'.format(thumbbase, ext) + shutil.copy2(os.path.join(base, thumbpath_rel), + os.path.join(workbase, thumbname)) + stream['poster'] = thumbname + else: + thumbjob = pool.apply_async(_Worker.make_thumb, + (self.worker, + streampath_abs, + workbase)) + transcodes['poster'] = thumbjob + + pending.append({'jobs': transcodes, + 'data': stream}) pool.close() pool.join() @@ -89,16 +87,10 @@ class Transcoder: for item in pending: stream = item['data'] jobs = item['jobs'] - slide_jobs = item.get('slides') streams = {} - slides = {} for maxheight in self.variants: streams[maxheight] = jobs[maxheight].get() - if slide_jobs is not None: - slides[maxheight] = slide_jobs[maxheight].get() stream['video'] = streams - if slide_jobs is not None: - stream['slides'] = slides if 'poster' in jobs: stream['poster'] = jobs['poster'].get() if not package['thumb'] and stream['playAudio']: -- 2.39.5 From fae0286f0db4316d593d5c1dae8471141d5cdc76 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 15:28:58 +0200 Subject: [PATCH 30/59] removed demux_file from stream --- daemon/transcoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index adb1445..b049794 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -47,6 +47,7 @@ class Transcoder: slides_thumb = 'slides_thumb{}'.format(ext) shutil.copy2(stream['poster'],os.path.join(workbase, slides_thumb)) stream['poster'] = slides_thumb + stream.pop('demux_file') elif 'video' in stream: streampath_rel = stream['video'] -- 2.39.5 From 129a1933cbb1603542975326ec45a05e7dfcb8c7 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 15:50:51 +0200 Subject: [PATCH 31/59] small changes --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 35854e8..3f406c3 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -76,7 +76,7 @@ class Mediasite: f.write('file \'{}\'\n'.format(slides[-1]['url'])) mypackage['sources'].append({ 'demux_file': demux_file, 'poster': slides[0]['url'], - 'playAudio': "false" }) + 'playAudio': False }) return mypackage def _download(self, base, remotefile, session): -- 2.39.5 From b595f3ccadb5fa73cf231960794bd4ea4ac9ff5b Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 15 Oct 2021 15:59:24 +0200 Subject: [PATCH 32/59] final clean/small fixes --- daemon/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/pipeline.py b/daemon/pipeline.py index 38f376d..3e1a740 100644 --- a/daemon/pipeline.py +++ b/daemon/pipeline.py @@ -126,6 +126,8 @@ class Pipeline: note = dict(package) if 'update_id' in note: note['id'] = note.pop('update_id') + if 'slides' in note: + note.pop('slides') del(note['base']) del(note['workbase']) result = requests.post(self.notify_url, -- 2.39.5 From 42f48e247d13885deed89b8bb911d069a85f21eb Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 18 Oct 2021 09:56:06 +0200 Subject: [PATCH 33/59] Changed framrates to see if get better sync --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index b049794..8a74db1 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', safe=0) + .input(demux_file, framrate=24, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From 8275129b59114d4e5bffd69eba91161d0ffccea3 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 18 Oct 2021 10:01:44 +0200 Subject: [PATCH 34/59] spellchecking anyone? --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 8a74db1..6a9bff9 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, framrate=24, format='concat', safe=0) + .input(demux_file, framerate=24, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From e413477e7e07032c1e2c3532a91b1dc8eb4ef06a Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 18 Oct 2021 10:12:07 +0200 Subject: [PATCH 35/59] could it be? --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 6a9bff9..13c835f 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, framerate=24, format='concat', safe=0) + .input(demux_file, format='concat', safe=0, framerate=25) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From c0a4149e25edfaac7dc215355c8f720015e0d5df Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 18 Oct 2021 11:24:55 +0200 Subject: [PATCH 36/59] testing new duration calculation --- daemon/packager.py | 6 ++++-- daemon/transcoder.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 3f406c3..c1ce760 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -65,12 +65,14 @@ class Mediasite: slides = [] demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: + elapsed = 0 for slide in data['slides']: myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': '{}ms'.format(slide['duration'])} + 'duration': '{}ms'.format(int(slide['duration'])-elapsed)} f.write('file \'{}\'\n'.format(myslide['url'])) - f.write('duration {}ms\n'.format(slide['duration'])) + f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) + elapsed = int(slide(['duration'])) # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(slides[-1]['url'])) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 13c835f..b049794 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', safe=0, framerate=25) + .input(demux_file, format='concat', safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From 69469df42be282a463a627572d6a6560b454ebad Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 18 Oct 2021 11:41:22 +0200 Subject: [PATCH 37/59] omg --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index c1ce760..608237e 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -72,7 +72,7 @@ class Mediasite: f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) - elapsed = int(slide(['duration'])) + elapsed = int(slide['duration']) # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(slides[-1]['url'])) -- 2.39.5 From 3104bbb2e5fe4bb738c527e9592037bd79d57e9b Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 09:40:30 +0200 Subject: [PATCH 38/59] trying adding start_time for video --- daemon/packager.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 608237e..4d65d1d 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -64,21 +64,28 @@ class Mediasite: os.mkdir(slides_path) slides = [] demux_file = os.path.join(slides_path, 'demux.txt') + start_time = timedelta(minutes=3, seconds=51, milliseconds=170) with open(demux_file, 'w') as f: elapsed = 0 - for slide in data['slides']: - myslide = { 'url': os.path.join(slides_path, self._download(slides_path, slide['url'], session)), - 'duration': '{}ms'.format(int(slide['duration'])-elapsed)} + myslide = {} + for i in range(len(data['slides'])): + myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slide'][i]['url'], session)) + if i == 0: + myslide['duration'] = '{}ms'.format((start_time.total_seconds * 1000) + int(data['slides'][i]['duration'])) + elapsed = start_time.total_seconds * 1000 + int(data['slides'][i]['duration']) + else: + myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration'])-elapsed) + elapsed = int(data['slides'][i]['duration']) + f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) - elapsed = int(slide['duration']) # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(slides[-1]['url'])) - mypackage['sources'].append({ 'demux_file': demux_file, - 'poster': slides[0]['url'], - 'playAudio': False }) + mypackage['sources'].append({'demux_file': demux_file, + 'poster': slides[0]['url'], + 'playAudio': False }) return mypackage def _download(self, base, remotefile, session): -- 2.39.5 From 28ae5b72dbf245708b66b65fe7bbb96c93ac3665 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 09:45:05 +0200 Subject: [PATCH 39/59] type --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 4d65d1d..d5db54d 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -69,7 +69,7 @@ class Mediasite: elapsed = 0 myslide = {} for i in range(len(data['slides'])): - myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slide'][i]['url'], session)) + myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: myslide['duration'] = '{}ms'.format((start_time.total_seconds * 1000) + int(data['slides'][i]['duration'])) elapsed = start_time.total_seconds * 1000 + int(data['slides'][i]['duration']) -- 2.39.5 From bbd8b3d99175b756a1815b765b278013128970d5 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 09:49:10 +0200 Subject: [PATCH 40/59] function not property --- daemon/packager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index d5db54d..37cb19c 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -71,8 +71,8 @@ class Mediasite: for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: - myslide['duration'] = '{}ms'.format((start_time.total_seconds * 1000) + int(data['slides'][i]['duration'])) - elapsed = start_time.total_seconds * 1000 + int(data['slides'][i]['duration']) + myslide['duration'] = '{}ms'.format((start_time.total_seconds() * 1000) + int(data['slides'][i]['duration'])) + elapsed = start_time.total_seconds() * 1000 + int(data['slides'][i]['duration']) else: myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration'])-elapsed) elapsed = int(data['slides'][i]['duration']) -- 2.39.5 From ce8e1f593d7b6e3c479216e7698009609db3fc7c Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 10:00:33 +0200 Subject: [PATCH 41/59] sukk --- daemon/packager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 37cb19c..0b5dfbe 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -64,19 +64,18 @@ class Mediasite: os.mkdir(slides_path) slides = [] demux_file = os.path.join(slides_path, 'demux.txt') - start_time = timedelta(minutes=3, seconds=51, milliseconds=170) + offset = timedelta(minutes=3, seconds=51, milliseconds=170).total_seconds() * 1000 with open(demux_file, 'w') as f: elapsed = 0 myslide = {} for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: - myslide['duration'] = '{}ms'.format((start_time.total_seconds() * 1000) + int(data['slides'][i]['duration'])) - elapsed = start_time.total_seconds() * 1000 + int(data['slides'][i]['duration']) + myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])) else: - myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration'])-elapsed) - elapsed = int(data['slides'][i]['duration']) + myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])-elapsed) + elapsed = offset + int(data['slides'][i]['duration']) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) -- 2.39.5 From 7d649fc6821614a48f8a1625417ad2bd1a0451e7 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 10:30:28 +0200 Subject: [PATCH 42/59] trying other calc --- daemon/packager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 0b5dfbe..c0016a2 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -66,16 +66,16 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') offset = timedelta(minutes=3, seconds=51, milliseconds=170).total_seconds() * 1000 with open(demux_file, 'w') as f: - elapsed = 0 + elapsed = 0 + offset myslide = {} for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: - myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])) + myslide['duration'] = '{}ms'.format(elapsed + int(data['slides'][i]['duration'])) + elapsed += int(data['slides'][i]['duration']) else: - myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])-elapsed) + myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration']) - int(data['slides'][i-1]['duration'])) - elapsed = offset + int(data['slides'][i]['duration']) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) -- 2.39.5 From 591fefcb058bf7fe6205d5c4318587808ecb0f49 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 11:01:59 +0200 Subject: [PATCH 43/59] removed float --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index c0016a2..a000d62 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -66,7 +66,7 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') offset = timedelta(minutes=3, seconds=51, milliseconds=170).total_seconds() * 1000 with open(demux_file, 'w') as f: - elapsed = 0 + offset + elapsed = 0 + int(offset) myslide = {} for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) -- 2.39.5 From 9fcaee51ad855a0a7abfdd080c8dc69a0a6c78b3 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Tue, 19 Oct 2021 14:31:00 +0200 Subject: [PATCH 44/59] testing segment_time_metadata --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index b049794..fa0412e 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', safe=0) + .input(demux_file, format='concat', safe=0, segment_time_metadata=1) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From 85cf9f28a13b3a6a8e74194683de949a2974c890 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 20 Oct 2021 11:18:46 +0200 Subject: [PATCH 45/59] added concat isntruction to demux.txt file, cahnged ffmpeg input call accordingly --- daemon/packager.py | 1 + daemon/transcoder.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index a000d62..8e66e05 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -68,6 +68,7 @@ class Mediasite: with open(demux_file, 'w') as f: elapsed = 0 + int(offset) myslide = {} + f.write('ffconcat version 1.0\n') for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: diff --git a/daemon/transcoder.py b/daemon/transcoder.py index fa0412e..e5bea2b 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -176,7 +176,7 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() (ffmpeg - .input(demux_file, format='concat', safe=0, segment_time_metadata=1) + .input(demux_file, safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') .overwrite_output() -- 2.39.5 From c4ab578d637380069414de002c9688d96eccefea Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Wed, 20 Oct 2021 16:39:15 +0200 Subject: [PATCH 46/59] tweaks and fixes. lets see --- daemon/packager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 8e66e05..f35d9e6 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -59,21 +59,20 @@ class Mediasite: 'playAudio': source['playAudio']} mypackage['sources'].append(mysource) + if 'slides' in data: slides_path = os.path.join(base, 'slides') os.mkdir(slides_path) slides = [] demux_file = os.path.join(slides_path, 'demux.txt') - offset = timedelta(minutes=3, seconds=51, milliseconds=170).total_seconds() * 1000 + offset = data['duration'] - int(data['slides'][-1]['duration']) with open(demux_file, 'w') as f: - elapsed = 0 + int(offset) myslide = {} f.write('ffconcat version 1.0\n') for i in range(len(data['slides'])): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == 0: - myslide['duration'] = '{}ms'.format(elapsed + int(data['slides'][i]['duration'])) - elapsed += int(data['slides'][i]['duration']) + myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])) else: myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration']) - int(data['slides'][i-1]['duration'])) -- 2.39.5 From 2cd9199ec459728ffbbc7e3ecb6401c9a3a6ed43 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 09:55:04 +0200 Subject: [PATCH 47/59] test removing options from output --- daemon/transcoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index e5bea2b..4b5093c 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -178,7 +178,7 @@ class _Worker: (ffmpeg .input(demux_file, safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) - .output(outpath, vcodec='libx264', crf=crf, preset=preset, vsync='vfr', movflags='+faststart', pix_fmt='yuv420p', c='copy') + .output(outpath, crf=crf, preset=preset, pix_fmt='yuv420p') .overwrite_output() .run(quiet=quiet)) -- 2.39.5 From 84c94ca6c56a361eba3dc6240b971f90492fbe0d Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 13:51:31 +0200 Subject: [PATCH 48/59] duration in package is timestamp when slide starts... --- daemon/packager.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index f35d9e6..4a672fc 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -65,17 +65,20 @@ class Mediasite: os.mkdir(slides_path) slides = [] demux_file = os.path.join(slides_path, 'demux.txt') - offset = data['duration'] - int(data['slides'][-1]['duration']) + + #offset = data['duration'] - int(data['slides'][-1]['duration']) with open(demux_file, 'w') as f: myslide = {} f.write('ffconcat version 1.0\n') - for i in range(len(data['slides'])): + num_slides = len(data['slides']) + for i in range(num_slides): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) - if i == 0: - myslide['duration'] = '{}ms'.format(offset + int(data['slides'][i]['duration'])) + if i == num_slides - 1: + myslide['duration'] = '{}ms'.format(data['duration'] - int(slides['data'][i]['duration'])) else: - myslide['duration'] = '{}ms'.format(int(data['slides'][i]['duration']) - int(data['slides'][i-1]['duration'])) + myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) + #offset += int(data[['slides'][i]['duration']]) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) -- 2.39.5 From 5048792a690c1a658ec5e2088c328f7e9f551e1c Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 13:55:56 +0200 Subject: [PATCH 49/59] fixes --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 4a672fc..9a28efc 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -74,7 +74,7 @@ class Mediasite: for i in range(num_slides): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == num_slides - 1: - myslide['duration'] = '{}ms'.format(data['duration'] - int(slides['data'][i]['duration'])) + myslide['duration'] = '{}ms'.format(data['duration'] - int(data['slides'][i]['duration'])) else: myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) -- 2.39.5 From 45a45621e33bfd24b82bf62c9e516dc3ab3c452f Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 14:17:21 +0200 Subject: [PATCH 50/59] edgecases? --- daemon/packager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/packager.py b/daemon/packager.py index 9a28efc..71e37f2 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -75,6 +75,8 @@ class Mediasite: myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == num_slides - 1: myslide['duration'] = '{}ms'.format(data['duration'] - int(data['slides'][i]['duration'])) + elif i == 0: + myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration'])) else: myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) -- 2.39.5 From 6306418038536e24148f71eaeebfc0089763f9c5 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 15:16:41 +0200 Subject: [PATCH 51/59] coments --- daemon/packager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index 71e37f2..445740d 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -5,7 +5,6 @@ import re import shutil import logging -import requests from requests.auth import HTTPBasicAuth from requests.sessions import Session @@ -49,7 +48,8 @@ class Mediasite: if 'id' in data: mypackage['notification_id'] = data['id'] - with requests.Session() as session: + # Create one session for all the downloads + with Session() as session: session.auth = self.auth session.stream = True mypackage['thumb'] = self._download(base, data['thumb'], session) @@ -66,11 +66,12 @@ class Mediasite: slides = [] demux_file = os.path.join(slides_path, 'demux.txt') - #offset = data['duration'] - int(data['slides'][-1]['duration']) with open(demux_file, 'w') as f: myslide = {} f.write('ffconcat version 1.0\n') num_slides = len(data['slides']) + + # loop all slides and download, calculate the duration and create a text file holding all the info for he ffmpeg demuxer for i in range(num_slides): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) if i == num_slides - 1: @@ -80,7 +81,6 @@ class Mediasite: else: myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) - #offset += int(data[['slides'][i]['duration']]) f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) @@ -88,10 +88,11 @@ class Mediasite: # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(slides[-1]['url'])) mypackage['sources'].append({'demux_file': demux_file, - 'poster': slides[0]['url'], + 'poster': slides[-1]['url'], 'playAudio': False }) return mypackage - + + # function to download the material from mediasite def _download(self, base, remotefile, session): localname = remotefile.split('/')[-1] localpath = os.path.join(base, localname) -- 2.39.5 From 345be98371529162939c6949bc13a32829578717 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 15:17:52 +0200 Subject: [PATCH 52/59] clean and comments --- daemon/packager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon/packager.py b/daemon/packager.py index 445740d..1b336a2 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -74,6 +74,8 @@ class Mediasite: # loop all slides and download, calculate the duration and create a text file holding all the info for he ffmpeg demuxer for i in range(num_slides): myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) + + # Handle the different edgecases for indiviual slide duration if i == num_slides - 1: myslide['duration'] = '{}ms'.format(data['duration'] - int(data['slides'][i]['duration'])) elif i == 0: @@ -81,6 +83,7 @@ class Mediasite: else: myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) + # Commit to the demuxfile f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) -- 2.39.5 From 10c46f9c1a437f25d07b09cc3d84d4eca456ffc7 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 15:44:23 +0200 Subject: [PATCH 53/59] more comments and cleaning --- daemon/packager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon/packager.py b/daemon/packager.py index 1b336a2..669366a 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -53,6 +53,8 @@ class Mediasite: session.auth = self.auth session.stream = True mypackage['thumb'] = self._download(base, data['thumb'], session) + + # Download video sources, and store the information with a local file path in mypackage for the transcoder for source in data['sources']: mysource = {'video': self._download(base, source['video'], session), 'poster': self._download(base, source['poster'], session), @@ -73,6 +75,8 @@ class Mediasite: # loop all slides and download, calculate the duration and create a text file holding all the info for he ffmpeg demuxer for i in range(num_slides): + + # Download the source file and change the url to a local file path myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) # Handle the different edgecases for indiviual slide duration @@ -90,6 +94,8 @@ class Mediasite: # to accomodate for an ffmpeg quirk that needs the last slide twice f.write('file \'{}\'\n'.format(slides[-1]['url'])) + + # put all the slides info in mypackage for the transcoder to rework into a mp4 video mypackage['sources'].append({'demux_file': demux_file, 'poster': slides[-1]['url'], 'playAudio': False }) -- 2.39.5 From e7c81318f603b1b15bc3c8fd479e15d5d8dc9b31 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 15:45:16 +0200 Subject: [PATCH 54/59] comments --- daemon/transcoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 4b5093c..5f87565 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -163,6 +163,7 @@ class _Worker: package_id, outstream, runtime) return outstream + # Function to make a video from images used for the slideshow def make_slides_video(self, package_id, demux_file, outdir, maxheight, preset, crf): self.logger.debug("%s - Preparing slides for processing", package_id) outstream = '{}-{}-{}-{}.mp4'.format('slides', maxheight, preset, crf) -- 2.39.5 From d31691b67d5888a64ca690cc42e742e803fa42da Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 15:51:13 +0200 Subject: [PATCH 55/59] more comments --- daemon/transcoder.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 5f87565..6cc7087 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -33,10 +33,16 @@ class Transcoder: self.logger.debug("%s - Pool created", package_id) for stream in package['sources']: transcodes = {} + + # If the strean is a colection of slide images that has to be converted to a video if 'demux_file' in stream: self.logger.debug("%s - Processing stream %s", package_id, 'slides job in demux.txt') + + # Create the different variants for maxheight in self.variants: crf, preset = self.variants[maxheight] + + # Call the _Worker function asyncronously transcodes[maxheight] = pool.apply_async(_Worker.make_slides_video, (self.worker, package_id, @@ -47,14 +53,21 @@ class Transcoder: slides_thumb = 'slides_thumb{}'.format(ext) shutil.copy2(stream['poster'],os.path.join(workbase, slides_thumb)) stream['poster'] = slides_thumb - stream.pop('demux_file') + # Remove the reference to the demuxfile since it is no longer needed + stream.pop('demux_file') + + # If the strean is a regualr video it needs to be transcoded with new resolution elif 'video' in stream: streampath_rel = stream['video'] streampath_abs = os.path.join(base, streampath_rel) self.logger.debug("%s - Processing stream %s", package_id, streampath_rel) + + # Create the different variants for maxheight in self.variants: crf, preset = self.variants[maxheight] + + # Call the _Worker function asyncronously transcodes[maxheight] = pool.apply_async(_Worker.transcode, (self.worker, package_id, @@ -77,9 +90,11 @@ class Transcoder: workbase)) transcodes['poster'] = thumbjob + # Store the jobs and streams in the pending array pending.append({'jobs': transcodes, 'data': stream}) + # Close the pool pool.close() pool.join() self.logger.info("%s - Finished transcoding", package_id) @@ -90,6 +105,7 @@ class Transcoder: jobs = item['jobs'] streams = {} for maxheight in self.variants: + # Get the jobs. If not done wait until they are ready. streams[maxheight] = jobs[maxheight].get() stream['video'] = streams if 'poster' in jobs: @@ -98,6 +114,7 @@ class Transcoder: package['thumb'] = stream['poster'] package['sources'].append(stream) + # Return the finished streams return package class _Worker: -- 2.39.5 From b6217985f21c4bea876300282c49ca7b6e4d23a3 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Thu, 21 Oct 2021 16:16:21 +0200 Subject: [PATCH 56/59] further commenting --- daemon/packager.py | 4 +++- daemon/transcoder.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 669366a..da00ab5 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -87,7 +87,9 @@ class Mediasite: else: myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) - # Commit to the demuxfile + # Commit to the demuxfile. Duration is assumed to be seconds unless staded otherwise + # https://ffmpeg.org/ffmpeg-utils.html#Time-duration + # https://trac.ffmpeg.org/wiki/Slideshow f.write('file \'{}\'\n'.format(myslide['url'])) f.write('duration {}\n'.format(myslide['duration'])) slides.append(myslide) diff --git a/daemon/transcoder.py b/daemon/transcoder.py index 6cc7087..f00792f 100644 --- a/daemon/transcoder.py +++ b/daemon/transcoder.py @@ -193,6 +193,8 @@ class _Worker: self.logger.debug("%s - Calling ffmpeg", package_id) start = time.time() + # https://ffmpeg.org/ffmpeg-formats.html#toc-concat-1 + # https://github.com/kkroening/ffmpeg-python/tree/master/examples (ffmpeg .input(demux_file, safe=0) .filter('scale', height='min(in_h, {})'.format(maxheight), width=-2) -- 2.39.5 From 808cbcfb3cda3ce2680315a32759f7f943a41bbb Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 22 Oct 2021 09:38:19 +0200 Subject: [PATCH 57/59] more commenting --- daemon/packager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index da00ab5..ce95d22 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -61,7 +61,7 @@ class Mediasite: 'playAudio': source['playAudio']} mypackage['sources'].append(mysource) - + # Slides exist, create a package for creating a video from the slides if 'slides' in data: slides_path = os.path.join(base, 'slides') os.mkdir(slides_path) @@ -76,15 +76,15 @@ class Mediasite: # loop all slides and download, calculate the duration and create a text file holding all the info for he ffmpeg demuxer for i in range(num_slides): - # Download the source file and change the url to a local file path + # Download the source file and store the url as a local file path myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) # Handle the different edgecases for indiviual slide duration - if i == num_slides - 1: + if i == num_slides - 1: # last slide myslide['duration'] = '{}ms'.format(data['duration'] - int(data['slides'][i]['duration'])) - elif i == 0: + elif i == 0: # first slide myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration'])) - else: + else: # all other slides myslide['duration'] = '{}ms'.format(int(data['slides'][i+1]['duration']) - int(data['slides'][i]['duration'])) # Commit to the demuxfile. Duration is assumed to be seconds unless staded otherwise @@ -99,7 +99,7 @@ class Mediasite: # put all the slides info in mypackage for the transcoder to rework into a mp4 video mypackage['sources'].append({'demux_file': demux_file, - 'poster': slides[-1]['url'], + 'poster': slides[0]['url'], 'playAudio': False }) return mypackage -- 2.39.5 From 23f740ae906c4aba6180af29f303ae33865550a2 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Fri, 22 Oct 2021 11:09:10 +0200 Subject: [PATCH 58/59] fixed obj bug where the same object is just updated with new data and then appended to array. meaning the array contains n obj referencing the same object. --- daemon/packager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/packager.py b/daemon/packager.py index ce95d22..0b159a7 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -69,7 +69,6 @@ class Mediasite: demux_file = os.path.join(slides_path, 'demux.txt') with open(demux_file, 'w') as f: - myslide = {} f.write('ffconcat version 1.0\n') num_slides = len(data['slides']) @@ -77,7 +76,7 @@ class Mediasite: for i in range(num_slides): # Download the source file and store the url as a local file path - myslide['url'] = os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session)) + myslide = { 'url': os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session))} # Handle the different edgecases for indiviual slide duration if i == num_slides - 1: # last slide -- 2.39.5 From 0ceb83e0a418dc9cd639754eeef694742cb5d500 Mon Sep 17 00:00:00 2001 From: daffyDukk Date: Mon, 25 Oct 2021 11:25:08 +0200 Subject: [PATCH 59/59] clean --- daemon/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/packager.py b/daemon/packager.py index 0b159a7..1374c5a 100644 --- a/daemon/packager.py +++ b/daemon/packager.py @@ -74,7 +74,7 @@ class Mediasite: # loop all slides and download, calculate the duration and create a text file holding all the info for he ffmpeg demuxer for i in range(num_slides): - + # Download the source file and store the url as a local file path myslide = { 'url': os.path.join(slides_path, self._download(slides_path, data['slides'][i]['url'], session))} -- 2.39.5