Re: Anyone else having problems with commercial removal in 0
Posted: Thu Nov 24, 2016 4:45 am
Messaged you what I'd changed so you have it.
The Official Forums for discussions of all things related to the MythTV Media Center
https://forum.mythtv.org/
Code: Select all
def prog_check():
#checks for FFmpeg/FFprobe
#if not found MythFFmpeg/MythFFprobe is used
from distutils import spawn
if spawn.find_executable('ffmpeg') != None:
ffmpeg = spawn.find_executable('ffmpeg')
if (spawn.find_executable('ffmpeg') == None and
spawn.find_executable('mythffmpeg')):
ffmpeg = spawn.find_executable('mythffmpeg')
else:
ffmpeg = None
if spawn.find_executable('ffprobe') != None:
ffprobe = spawn.find_executable('ffprobe')
if (spawn.find_executable('ffprobe') == None and
spawn.find_executable('mythffprobe')):
ffprobe = spawn.find_executable('mythffprobe')
else:
ffprobe = None
if ffmpeg != None and ffprobe != None:
return ffmpeg,ffprobe
else:
raise LookupError('Unable to find FFmpeg or MythFFmpeg')
Code: Select all
cut =subprocess.Popen([ffmpeg,'-ignore_unknown','-i',infile,'-c','copy','-map','0','-f','segment','-segment_list','%scut.ffcat'%(tmpdir),'-segment_frames',cutlist,'-break_non_keyframes', '1','%scut%%03d.ts'%(tmpdir)],stdout = subprocess.PIPE, stderr = subprocess.STDOUT,universal_newlines=True)
Code: Select all
ffprobe -shows_streams file.ts
Code: Select all
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
#MythTV commercial removal -- by Daraden
#usage userjob = /path to script/script.py --jobid %JOBID%
#/path to script/script.py --chanid --starttime can also be used
from __future__ import print_function
from MythTV import (Job, Program, Recorded, System, MythDB, findfile,
MythError, MythLog, datetime, VideoGrabber)
import argparse
import shutil
from glob import glob
import time
from datetime import timedelta
import subprocess
import ConfigParser
import logging
import logging.handlers
import sys
import os
import tempfile
# set this to True to use commflag results as cut-list
# this will automaticly flag commercials if no cut-list is available
use_commflag = True
# save copy of original file as file.old if not using output file
save_old = False
# user needs write access to this directory to save logfile
logdir ='/media/windows/test/'
LOG_FILENAME = '{}cut.log'.format(logdir)
# logging setup for file and console
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
lf= logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(lf)
logger.addHandler(ch)
try:
if os.access(logdir,os.W_OK):
fh = logging.handlers.TimedRotatingFileHandler(filename=LOG_FILENAME,
when='W0',interval=1,
backupCount=10
)
fh.setLevel(logging.DEBUG)
fh.setFormatter(lf)
logger.addHandler(fh)
if not os.access(logdir,os.W_OK):
logging.error('logdir not accessible:%s',logdir)
except exception as e:
logging.error('logdir not accessible:%s',logdir)
logging.error(e)
sys.exit(1)
try:
db=MythDB()
except exception as e:
logging.error(e)
sys.exit(1)
def prog_check():
"""checks for FFmpeg/FFprobe if not found MythFFmpeg/MythFFprobe
is used"""
from distutils import spawn
if spawn.find_executable('ffmpeg') != None:
ffmpeg = spawn.find_executable('ffmpeg')
elif (spawn.find_executable('ffmpeg') == None and
spawn.find_executable('mythffmpeg')):
ffmpeg = spawn.find_executable('mythffmpeg')
else:
ffmpeg = None
if spawn.find_executable('ffprobe') != None:
ffprobe = spawn.find_executable('ffprobe')
elif (spawn.find_executable('ffprobe') == None and
spawn.find_executable('mythffprobe')):
ffprobe = spawn.find_executable('mythffprobe')
else:
ffprobe = None
if ffmpeg != None and ffprobe != None:
return ffmpeg,ffprobe
else:
raise LookupError('Unable to find FFmpeg or MythFFmpeg')
#check for ffmpeg or mythffmpeg
ffmpeg = prog_check()[0]
logging.debug('ffmpeg={}'.format(ffmpeg))
ffprobe = prog_check()[1]
logging.debug('ffprobe={}'.format(ffprobe))
def run_cut(jobid=None,chanid=None,starttime=None,outfile=None):
logging.info('Started')
# Configure chanid and starttime from userjob input
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
starttime = job.starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
if not jobid:
chanid = chanid
starttime = starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
# Get database recording entry
rec = find_rec(chanid,starttime)
logging.debug('DB recording entry={}'.format(rec))
# Find and format full input file path
sg = findfile('/{}'.format(rec.basename), rec.storagegroup, db=db)
infile = os.path.join(sg.dirname, rec.basename)
# Assign and create temporary directory in recording directory
tmpdir = '{}{}{}/'.format(sg.dirname, 'crtmp.',
rec.basename.split('.')[0]
)
logging.debug('tmpdir is: {}'.format(tmpdir))
tmp_chk(tmpdir)
# Get info from input file
avinfo = AVInfo(ffprobe,infile)
# Get video frame-rate
rfr = list(int(x) for x in avinfo.video.r_frame_rate.split('/'))
frame_rate = rfr[0] / float(rfr[1])
logging.info('Video frame-rate is: {}'.format(frame_rate))
if save_old == True and outfile == None:
logging.info('Backing up original file: %s',infile)
shutil.copyfile(infile,'%s.old' % infile)
logging.info('Finished backing up original file: {}'.format(infile))
# Configure output file
if outfile != None:
Outfile = outfile
elif outfile == None:
Outfile = infile
if rec.cutlist == 1 or use_commflag == True:
##Flag commercials##
if use_commflag == True and rec.commflagged != 1 and rec.cutlist != 1:
try:
logging.info('Flagging commercials')
startts = datetime.mythformat(rec.starttime)
rst = subprocess.Popen(['mythcommflag','--chanid',
str(chanid),'--starttime',
str(startts)],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Commercials flagged for:%s_%s',
chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Flagging '
'commercials for:{}_{}'
.format(chanid,starttime)
)
logging.error('MythCommflag allways exits with '
'decoding error?!'
)
# uncoment folowing line to log mythcommflag errors
#logging.error('{}'.format(output2))
except Exception as e:
logging.error('Mythcommflag ERROR: Flagging commercials %s',e)
# Reinitalize recording object
rec = find_rec(chanid,starttime)
if jobid:
job.update(
{'status':job.RUNNING, 'comment':'Removing Cutlist'})
logging.info('getting cutlist')
# Retrieve cutlist from database
print(type(rec.cutlist))
if rec.cutlist == 1:
cut_list = (','.join(
[','.join(str(i) for i in b)
for b in rec.markup.getcutlist()]))
print(cut_list)
elif rec.cutlist != 1 and rec.commflagged == 1 and use_commflag == True:
cut_list = (','.join([','.join(str(i) for i in b)
for b in rec.markup.getskiplist()]))
if cut_list == '':
logging.error('Commflag list empty: No commercials flaged!')
if jobid:
job.update(
{'status':job.ERRORED, 'comment':'Commflag list empty'})
sys.exit(1)
else:
logging.debug('No cut/skip list found')
sys.exit(1)
cutlist = cut_list
print(cutlist)
# Format cutlist
if cutlist.startswith('0,'):
cutlist = cutlist.replace("0,",'',1)
if cutlist.endswith(',9999999'):
cutlist = cutlist.replace(",9999999",'',-1)
logging.debug('Myth cutlist:{}'.format(cut_list))
logging.debug('FFmpeg cutlist:{}'.format(cutlist))
logging.info('cutting started')
cut(ffmpeg, infile,cutlist, tmpdir)
logging.info('Merging Cuts')
avinfo = AVInfo(ffprobe,infile)
join(ffmpeg, cutlist, tmpdir, Outfile, avinfo)
rem_tmp(tmpdir)
if outfile == None:
clear_markup(rec,infile,outfile)
if jobid:
job.update({'status':job.FINISHED, 'comment':
'Commercial removal Completed'
}
)
class DictToNamespace(object):
""" convert a dictonary and any nested dictonarys to namespace"""
def __setitem__(self, key, item):
self.__dict__[key] = item
def __getitem__(self, key):
return self.__dict__[key]
def has_key(self, k):
return self.__dict__.has_key(k)
def update(self, *args, **kwargs):
return self.__dict__.update(*args, **kwargs)
def keys(self):
return self.__dict__.keys()
def values(self):
return self.__dict__.values()
def items(self):
return self.__dict__.items()
def __contains__(self, item):
return item in self.__dict__
def __iter__(self):
return iter(self.__dict__)
def __init__(self, data):
for item,num in data.items():
if isinstance(num, str):
try:
num = float(num)
if float(num).is_integer():
num = int(num)
data.update({item: num})
except ValueError:
pass
self.__dict__.update(data)
for k, v in data.items():
if isinstance(v, dict):
self.__dict__[k] = DictToNamespace(v)
class AVInfo:
"""identify A/V configuration of input file and returns
self.video dict a list of self.audio.stream dicts,
and self.duration as float"""
def __init__(self=None,ffprobe=None,infile=None):
class Dict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
command = [ffprobe, '-v', '-8', '-show_entries',
'stream=codec_type,index,codec_name,channels,width,'
'height,r_frame_rate:stream_tags=language:'
'format=duration', '-of', 'csv=nk=0:p=0', infile
]
x = subprocess.check_output(command).decode('utf-8').split('\n')
vcd = {}
adict = {}
for vc in x:
if 'codec_type=video' in vc:
for item in vc.split(','):
k,v = item.split('=')
vcd.update({k:v})
for ac in x:
if 'codec_type=audio' in ac:
items =[item.split('=') for item in ac.split(',')]
streamdict ={k.strip(): v.strip() for (k,v) in items}
if 'tag:language' in streamdict.keys():
streamdict['language'] = (streamdict.pop
('tag:language')
)
adict.update({'stream{}'.format(streamdict['index'])
: streamdict})
for d in x:
if d.startswith('duration='):
dur = d.split('=')[1]
self.video = Dict(vcd)
self.audio = DictToNamespace(adict)
self.duration = dur
def cut(ffmpeg, infile,cutlist, tmpdir):
"""use ffmpeg segment muxer to cut infile into multipule files
using a csv formated cutlist placing them into tmpdir"""
cmd = [ffmpeg, '-ignore_unknown', '-i', infile, '-c', 'copy',
'-map', '0'
]
av_check = AVInfo(ffprobe,infile)
for streams,stream in av_check.audio.items():
if stream.channels == '0':
cmd.extend(('-map',
('-0:{}'
.format(stream.index))))
cmd.extend(('-f','segment',
'-segment_frames', cutlist,'-break_non_keyframes', '1',
'{}cut%03d.ts'.format(tmpdir)
)
)
encode(cmd,av_check,ffmpeg)
def encode(command,AVInfo,ffmpeg):
""" Run ffmpeg command with status output"""
# Length of progress bar
statlen = 10
# Charicter used for progress bar
# Use chr() in python 3
statchar = unichr(9619).encode('UTF-8')
# Charicter used to pad progress bar
pad = ' '
rfr = list(int(x) for x in AVInfo.video.r_frame_rate.split('/'))
frame_rate = float(rfr[0]) / float(rfr[1])
duration = float(AVInfo.duration)
total_frames = duration * frame_rate
with tempfile.TemporaryFile() as output:
process = subprocess.Popen(command, stdout=output,
stderr =output,
universal_newlines=True
)
while True:
if process.poll() is not None:
if process.poll() != 0:
output.seek(0)
print(output.read().decode('UTF-8'))
sys.exit(1)
if process.poll() == 0:
print('\rFinished{}'.format(pad * (statlen + 3)))
break
where = output.tell()
lines = output.read().decode('UTF-8')
if not lines:
time.sleep(0.1)
output.seek(where)
elif lines.startswith('frame='):
ln = ' '.join(lines.split()).replace('= ','=').split(' ')
for item in ln:
if item.startswith('frame='):
framenum = int(item.replace('frame=',''))
if item.startswith('fps='):
fps = float(item.replace('fps=',''))
if int(framenum) == 0:
pcomp = 0
else:
# python 2 div
pcomp = 100 * (float(framenum) / float(total_frames))
#python 3 div
#pcomp = 100 * (framenum / total_frames)
# python 2 div
stat = int((float(pcomp) / float(100)) * statlen)
# python 3 div
#stat = int((int(pcomp) / 100) * statlen)
padlen = statlen - stat
status = "|{:6.2f}%|".format(pcomp)
statusbar = '|{}{}|'.format(statchar * stat, pad * padlen)
status = '\r{}{}'.format(status,statusbar)
print(status, end="")
# Replace with flush=True in print function for python 3
sys.stdout.flush()
def join(ffmpeg, cutlist, tmpdir, tmpfile,avinfo):
""" use ffmpeg to join video segments in tempdir into single tempfile.
using the first value of the cutlist to determine the segments.
avinfo object from source file used to determine frame-rate for
status output."""
j = []
durlist= []
# Length of progress bar
statlen = 10
# Charicter used for progress bar
# Use chr() in python 3
statchar = unichr(9619).encode('UTF-8')
# Charicter used to pad progress bar
pad = ' '
# Get list of segment files
for root, dirs, files in os.walk(tmpdir):
for files in files:
if (files.endswith('.ts') and
files.startswith('cut')):
if os.path.isfile(os.path.join(root,files)):
j.append(os.path.join(root,files))
# Create concat string and file list
if cutlist.startswith('0'):
filelist = j[0::2]
concatlist = ','.join(j[0::2]).replace(',','|')
if not cutlist.startswith('0'):
filelist = j[1::2]
concatlist = ','.join(j[1::2]).replace(',','|')
# Get duration of file segments for output
for files in filelist:
durlist.append(float(AVInfo(ffprobe,files).duration.strip('\r')))
duration = sum(durlist)
# format refrence framerate to float
rfr = list(int(x) for x in avinfo.video.r_frame_rate.split('/'))
frame_rate = rfr[0] / rfr[1]
# estimate total number of frames
total_frames = duration * frame_rate
command = [ffmpeg, '-y', '-i', 'concat:{}'.format(concatlist),
'-map', '0', '-c', 'copy', '-f', 'mpegts', tmpfile
]
with tempfile.TemporaryFile() as output:
process = subprocess.Popen(command, stdout=output,
stderr =output,
universal_newlines=True
)
while True:
if process.poll() is not None:
if process.poll() != 0:
output.seek(0)
print(output.read().decode('UTF-8'))
sys.exit(1)
if process.poll() == 0:
if os.path.isfile(tmpfile):
for file in j:
os.remove(file)
print('\rFinished{}'.format(pad * (statlen + 3)))
break
where = output.tell()
lines = output.read().decode('UTF-8')
if not lines:
time.sleep(0.1)
output.seek(where)
elif lines.startswith('frame='):
ln = ' '.join(lines.split()).replace('= ','=').split(' ')
for item in ln:
if item.startswith('frame='):
framenum = int(item.replace('frame=',''))
if item.startswith('fps='):
fps = float(item.replace('fps=',''))
if int(framenum) == 0:
pcomp = 0
else:
# python 2 div
pcomp = 100 * (float(framenum) / float(total_frames))
# python 3 div
#pcomp = 100 * (framenum / total_frames)
# python 2 div
stat = int((float(pcomp) / float(100)) * statlen)
# python 3 div
#stat = int((int(pcomp) / 100) * statlen)
padlen = statlen - stat
status = "|{:6.2f}%|".format(pcomp)
statusbar = '|{}{}|'.format(statchar * stat, pad * padlen)
status = '\r{}{}'.format(status,statusbar)
print(status, end="")
# Replace with flush=True in print function for python 3
sys.stdout.flush()
def clear_markup(rec,infile,outfile):
logging.info('Started Clearing markup')
logging.debug('rec=%s infile=%s outfile=%s',rec,infile,outfile)
chanid = rec.chanid
utcstarttime = rec.starttime
starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', ''))
logging.debug('chanid=%s starttime=%s',chanid,starttime)
logging.info('start clearing markup')
try:
rcl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearcutlist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output1 = rcl.communicate()
if rcl.wait() == 0:
logging.info('Cutlist removed for:%s_%s',chanid,starttime)
if rcl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing cutlist for:%s_%s',chanid,starttime)
logging.error('%s',output1)
except Exception as e:
logging.error('Mythutil exception clearing cutlist%s',e)
try:
rsl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearskiplist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rsl.communicate()[0]
if rsl.wait() == 0:
logging.info('Skiplist removed for:%s_%s',chanid,starttime)
if rsl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing skiplist for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythutil exception clearing skiplist:%s',e)
for index,mark in reversed(list(enumerate(rec.markup))):
if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END):
del rec.markup[index]
rec.bookmark = 0
rec.bookmarkupdate = datetime.now()
rec.cutlist = 0
rec.commflagged = 0
rec.markup.commit()
rec.basename = os.path.basename(infile)
rec.filesize = os.path.getsize(infile)
rec.transcoded = 1
rec.seek.clean()
rec.update()
try:
logging.info('Removing PNG files')
for png in glob('%s*.png' % infile):
os.remove(png)
except Exception as e:
logging.error('Error removing png files',e)
try:
logging.info('Removing JPG files')
for jpg in glob('%s*.jpg' % infile):
os.remove(jpg)
except Exception as e:
logging.error('Error removing jpg files',e)
try:
logging.info('Rebuilding seektable')
rst = subprocess.Popen(['mythcommflag','--chanid',str(chanid),'--starttime',str(starttime),'--rebuild'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Seektable Rebuilt for:%s_%s',chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Rebuilding Seektable for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythcommflag ERROR clearing skiplist:%s',e)
def tmp_chk(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.info('chk:temp Folder found: %s',tmpdir)
if os.listdir(tmpdir) != 0:
logging.warning('chk:Temp folder not empty!:Removing Files: %s',tmpdir)
shutil.rmtree(tmpdir)
os.makedirs(tmpdir)
if not os.path.isdir(tmpdir):
logging.info('chk:no temp folder found: %s',tmpdir)
os.makedirs(tmpdir)
logging.info('chk:Temp folder created: %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def rem_tmp(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.info('rem:temp Folder found %s',tmpdir)
shutil.rmtree(tmpdir)
if not os.path.isdir(tmpdir):
logging.info('rem:temp Folder Removed %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def find_rec(chanid,starttime):
def local_time_offset(t=None):
if t is None:
t = time.time()
if time.localtime(t).tm_isdst and time.daylight:
return -time.altzone
else:
return -time.timezone
def RecordedFromBasename(chanid,starttime):
bnts = '%s_%s.ts' % (chanid,starttime)
bnmpg = '%s_%s.mpg' % (chanid,starttime)
x = list(db.searchRecorded(basename=bnmpg))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
x = list(db.searchRecorded(basename=bnts))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
raise LookupError('unable to find Recorded entry for '
'ChanID {} StartTime {}'
.format(chanid, starttime)
)
try:
rec = Recorded((chanid,starttime), db=db)
except:
try:
tzoffset = local_time_offset() / (60*60)
utcstarttime = datetime.strptime(starttime,"%Y%m%d%H%M%S")
utcstarttime = utcstarttime + timedelta(hours=tzoffset)
rec = Recorded((chanid,utcstarttime), db=db)
except:
rec = RecordedFromBasename(chanid,starttime)
return rec
def main():
parser = argparse.ArgumentParser(description='MythTV Commercial removal and closed caption extraction tool.\nNOTE:Having .srt file in same dir as media files breaks playback in some media players(VLC)')
parser.add_argument('--chanid',action='store',type=str,dest='chanid',help='Channel-Id of Recording')
parser.add_argument('--starttime',action='store',type=str,dest='starttime',help='Starttime of recording in utc format')
parser.add_argument('--jobid',action='store',type=int,dest='jobid',help='JOBID')
parser.add_argument('-o',action='store',type=str,dest='outfile',help='Output file to be created')
args = parser.parse_args()
if args.jobid:
run_cut(jobid=args.jobid,outfile=args.outfile)
sys.exit(0)
if args.chanid and args.starttime:
run_cut(chanid=args.chanid,starttime=args.starttime,outfile=args.outfile)
sys.exit(0)
else:
print('chanid and starttime or jobid required')
main()
Code: Select all
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
#MythTV commercial removal -- by Daraden
#usage userjob = /path to script/script.py --jobid %JOBID%
#/path to script/script.py --chanid --starttime can also be used
from __future__ import print_function
from MythTV import (Job, Program, Recorded, System, MythDB, findfile,
MythError, MythLog, datetime, VideoGrabber)
import argparse
import shutil
from glob import glob
import time
from datetime import timedelta
import subprocess
import ConfigParser
import logging
import logging.handlers
import sys
import os
import tempfile
# set this to True to use commflag results as cut-list
# this will automaticly flag commercials if no cut-list is available
use_commflag = True
# save copy of original file as file.old if not using output file
save_old = False
# user needs write access to this directory to save logfile
logdir ='/media/windows/test/'
LOG_FILENAME = '{}cut.log'.format(logdir)
# logging setup for file and console
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
lf= logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(lf)
logger.addHandler(ch)
try:
if os.access(logdir,os.W_OK):
fh = logging.handlers.TimedRotatingFileHandler(filename=LOG_FILENAME,
when='W0',interval=1,
backupCount=10
)
fh.setLevel(logging.DEBUG)
fh.setFormatter(lf)
logger.addHandler(fh)
if not os.access(logdir,os.W_OK):
logging.error('logdir not accessible:%s',logdir)
except exception as e:
logging.error('logdir not accessible:%s',logdir)
logging.error(e)
sys.exit(1)
try:
db=MythDB()
except exception as e:
logging.error(e)
sys.exit(1)
def prog_check():
"""checks for FFmpeg/FFprobe if not found MythFFmpeg/MythFFprobe
is used"""
from distutils import spawn
if spawn.find_executable('ffmpeg') != None:
ffmpeg = spawn.find_executable('ffmpeg')
elif (spawn.find_executable('ffmpeg') == None and
spawn.find_executable('mythffmpeg')):
ffmpeg = spawn.find_executable('mythffmpeg')
else:
ffmpeg = None
if spawn.find_executable('ffprobe') != None:
ffprobe = spawn.find_executable('ffprobe')
elif (spawn.find_executable('ffprobe') == None and
spawn.find_executable('mythffprobe')):
ffprobe = spawn.find_executable('mythffprobe')
else:
ffprobe = None
if ffmpeg != None and ffprobe != None:
return ffmpeg,ffprobe
else:
raise LookupError('Unable to find FFmpeg or MythFFmpeg')
#check for ffmpeg or mythffmpeg
ffmpeg = prog_check()[0]
logging.debug('ffmpeg={}'.format(ffmpeg))
ffprobe = prog_check()[1]
logging.debug('ffprobe={}'.format(ffprobe))
def run_cut(jobid=None,chanid=None,starttime=None,outfile=None):
logging.info('Started')
# Configure chanid and starttime from userjob input
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
starttime = job.starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
if not jobid:
chanid = chanid
starttime = starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
# Get database recording entry
rec = find_rec(chanid,starttime)
logging.debug('DB recording entry={}'.format(rec))
# Find and format full input file path
sg = findfile('/{}'.format(rec.basename), rec.storagegroup, db=db)
infile = os.path.join(sg.dirname, rec.basename)
# Assign and create temporary directory in recording directory
tmpdir = '{}{}{}/'.format(sg.dirname, 'crtmp.',
rec.basename.split('.')[0]
)
logging.debug('tmpdir is: {}'.format(tmpdir))
logging.info('Creating temporary directory')
tmp_chk(tmpdir)
# Get info from input file
avinfo = AVInfo(ffprobe,infile)
# Get video frame-rate
rfr = list(int(x) for x in avinfo.video.r_frame_rate.split('/'))
frame_rate = rfr[0] / float(rfr[1])
logging.debug('Video frame-rate is: {}'.format(frame_rate))
if save_old == True and outfile == None:
if jobid:
job.update({'status':job.RUNNING,
'comment':'Backing up original file'
}
)
logging.info('Backing up original file: %s',infile)
shutil.copyfile(infile,'%s.old' % infile)
logging.info('Finished backing up original file: {}'.format(infile))
# Configure output file
if outfile != None:
Outfile = outfile
elif outfile == None:
Outfile = infile
if rec.cutlist == 1 or use_commflag == True:
##Flag commercials##
if (use_commflag == True and rec.commflagged != 1
and rec.cutlist != 1
):
# need check for running commflag job?
try:
logging.info('Flagging commercials')
if jobid:
job.update({'status':job.RUNNING,
'comment': 'Flagging commercials'
}
)
startts = datetime.mythformat(rec.starttime)
rst = subprocess.Popen(['mythcommflag','--chanid',
str(chanid),'--starttime',
str(startts)],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Commercials flagged for:%s_%s',
chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Flagging '
'commercials for:{}_{}'
.format(chanid,starttime)
)
logging.error('MythCommflag allways exits with '
'decoding error?!'
)
# uncoment folowing line to log mythcommflag errors
#logging.error('{}'.format(output2))
except Exception as e:
logging.error('Mythcommflag ERROR: Flagging commercials %s',e)
# Reinitalize recording object
rec = find_rec(chanid,starttime)
if jobid:
job.update({'status':job.RUNNING,
'comment':'Getting Cutlist'
}
)
logging.info('getting cutlist')
# Get cut/skip list from database
if rec.cutlist == 1:
myth_cut_list = rec.markup.getcutlist()
if rec.cutlist != 1 and rec.commflagged == 1 and use_commflag:
myth_cut_list = rec.markup.getskiplist()
elif rec.cutlist == 0 and rec.commflagged != 1 and not use_commflag:
logging.debug('No cut/skip list found')
sys.exit(1)
logging.debug('Myth cut_list: {}'.format(myth_cut_list))
# Format cutlist, removing starting 0 and ending 9999999
cut_list = [mark for cuts in myth_cut_list for mark in cuts]
if cut_list[0] == 0:
cut_list.pop(0)
if cut_list[-1] == 9999999:
cut_list.pop(-1)
logging.debug('Cut list: {}'.format(cut_list))
# Get the name of the video codec
video_codec = avinfo.video.codec_name
logging.info('video codec: {}'.format(video_codec))
# Set cutlist string
if video_codec == 'h264':
cut_string = ','.join(str(i / frame_rate) for i in cut_list)
elif video_codec == 'mpeg2video':
cut_string = ','.join(str(i) for i in cut_list)
logging.debug('cut_string: {}'.format(cut_string))
# Create segment files
logging.info('Cutting')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Cutting started'
}
)
cut(ffmpeg, infile, cut_string, tmpdir)
# Create list of files to join
file_list = []
# Get list of segment files
for root, dirs, files in os.walk(tmpdir):
for File in files:
if (File.endswith('.ts') and
File.startswith('cut')):
if os.path.isfile(os.path.join(root,File)):
file_list.append(os.path.join(root,File))
# Set list of files to be joined
if myth_cut_list[0][0] == 0:
join_list = file_list[1::2]
if myth_cut_list[0][0] != 0:
join_list = file_list[0::2]
logging.debug('Join file list: {}'.format(join_list))
# Join segment files
logging.info('Joining segments')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Joining segments'
}
)
join(ffmpeg, join_list, outfile)
# Remove temporary directory
logging.info('removing temporary directory')
rem_tmp(tmpdir)
if outfile == None:
logging.info('clearing database markup')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Clearing database markup'
}
)
clear_markup(rec,infile,outfile)
if jobid:
job.update({'status':job.FINISHED, 'comment':
'Commercial removal finished'
}
)
logging.info('Finished')
class DictToNamespace(object):
""" convert a dictonary and any nested dictonarys to namespace"""
def __setitem__(self, key, item):
self.__dict__[key] = item
def __getitem__(self, key):
return self.__dict__[key]
def has_key(self, k):
return self.__dict__.has_key(k)
def update(self, *args, **kwargs):
return self.__dict__.update(*args, **kwargs)
def keys(self):
return self.__dict__.keys()
def values(self):
return self.__dict__.values()
def items(self):
return self.__dict__.items()
def __contains__(self, item):
return item in self.__dict__
def __iter__(self):
return iter(self.__dict__)
def __init__(self, data):
for item,num in data.items():
if isinstance(num, str):
try:
num = float(num)
if float(num).is_integer():
num = int(num)
data.update({item: num})
except ValueError:
pass
self.__dict__.update(data)
for k, v in data.items():
if isinstance(v, dict):
self.__dict__[k] = DictToNamespace(v)
class AVInfo:
"""identify A/V configuration of input file and returns
self.video dict a list of self.audio.stream dicts,
and self.duration as float"""
def __init__(self=None,ffprobe=None,infile=None):
class Dict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
command = [ffprobe, '-v', '-8', '-show_entries',
'stream=codec_type,index,codec_name,channels,width,'
'height,r_frame_rate:stream_tags=language:'
'format=duration', '-of', 'csv=nk=0:p=0', infile
]
x = subprocess.check_output(command).decode('utf-8').split('\n')
vcd = {}
adict = {}
for vc in x:
if 'codec_type=video' in vc:
for item in vc.split(','):
k,v = item.split('=')
vcd.update({k:v})
for ac in x:
if 'codec_type=audio' in ac:
items =[item.split('=') for item in ac.split(',')]
streamdict ={k.strip(): v.strip() for (k,v) in items}
if 'tag:language' in streamdict.keys():
streamdict['language'] = (streamdict.pop
('tag:language')
)
adict.update({'stream{}'.format(streamdict['index'])
: streamdict})
for d in x:
if d.startswith('duration='):
dur = d.split('=')[1]
self.video = Dict(vcd)
self.audio = DictToNamespace(adict)
self.duration = dur
def encode(ffmpeg, command,AVInfo):
""" Run ffmpeg command with status output"""
# Length of progress bar
statlen = 10
# Charicter used for progress bar
# Use chr() in python 3
statchar = unichr(9619).encode('UTF-8')
# Charicter used to pad progress bar
pad = ' '
rfr = list(int(x) for x in AVInfo.video.r_frame_rate.split('/'))
frame_rate = float(rfr[0]) / float(rfr[1])
duration = float(AVInfo.duration)
total_frames = duration * frame_rate
with tempfile.TemporaryFile() as output:
process = subprocess.Popen(command, stdout=output,
stderr =output,
universal_newlines=True
)
while True:
if process.poll() is not None:
if process.poll() != 0:
output.seek(0)
print(output.read().decode('UTF-8'))
sys.exit(1)
if process.poll() == 0:
print('\rFinished{}'.format(pad * (statlen + 3)))
break
where = output.tell()
lines = output.read().decode('UTF-8')
if not lines:
time.sleep(0.1)
output.seek(where)
elif lines.startswith('frame='):
ln = ' '.join(lines.split()).replace('= ','=').split(' ')
for item in ln:
if item.startswith('frame='):
framenum = int(item.replace('frame=',''))
if item.startswith('fps='):
fps = float(item.replace('fps=',''))
if int(framenum) == 0:
pcomp = 0
else:
# python 2 div
pcomp = 100 * (float(framenum) / float(total_frames))
#python 3 div
#pcomp = 100 * (framenum / total_frames)
# python 2 div
stat = int((float(pcomp) / float(100)) * statlen)
# python 3 div
#stat = int((int(pcomp) / 100) * statlen)
padlen = statlen - stat
status = "|{:6.2f}%|".format(pcomp)
statusbar = '|{}{}|'.format(statchar * stat, pad * padlen)
status = '\r{}{}'.format(status,statusbar)
print(status, end="")
# Replace with flush=True in print function for python 3
sys.stdout.flush()
def cut(ffmpeg, infile, cutlist, tmpdir):
"""use ffmpeg segment muxer to cut infile into multipule files
using a csv formated cutlist placing them into tmpdir"""
cmd = [ffmpeg, '-ignore_unknown', '-i', infile, '-c', 'copy',
'-map', '0'
]
av_check = AVInfo(ffprobe,infile)
for streams,stream in av_check.audio.items():
if stream.channels == '0':
cmd.extend(('-map',
('-0:{}'
.format(stream.index))))
if av_check.video.codec_name == 'mpeg2video':
cmd.extend(('-f','segment',
'-segment_frames', cutlist,
'{}cut%03d.ts'.format(tmpdir)
)
)
if av_check.video.codec_name == 'h264':
cmd.extend(('-f','segment',
'-segment_times', cutlist,
'{}cut%03d.ts'.format(tmpdir)
)
)
encode(ffmpeg, cmd, av_check)
def join(ffmpeg, file_list, outfile):
# Create FFmpeg concat string
concat_string = ','.join(file_list).replace(',','|')
duration_list = []
rfr_list = []
video_codec_list = []
for files in file_list:
file_avinfo = AVInfo(ffprobe,files)
# get file duration
duration_list.append(float(file_avinfo.duration.strip('\r')))
# Get file refrence frame rate
rfr_list.append(file_avinfo.video.r_frame_rate.strip('\r'))
# Get file video codec
video_codec_list.append(file_avinfo.video.codec_name.strip('\r'))
# Duration of joined files
duration = sum(duration_list)
# Check that all files refrence framerate are equal
rfr_match = all(rfr_list[0] == item for item in rfr_list)
if len(rfr_list) > 0 and rfr_match:
# Convert refrence frame rate to float
rfr = rfr_list[0].split('/')
frame_rate = int(rfr[0]) * float(rfr[1])
elif len(rfr_list) == 0 or not rfr_match:
raise ValueError('Incorrect or missing refrence frame rate')
# Estimate total number of frames
total_frames = duration * frame_rate
#check if all files video codec match
codec_match = all(video_codec_list[0] == item
for item in video_codec_list
)
if codec_match:
video_codec = video_codec_list[0]
if not codec_match:
raise ValueError('Not all video codecs match')
command = [ffmpeg, '-y', '-i', 'concat:{}'.format(concat_string),
'-map', '0', '-c', 'copy', '-f', 'mpegts', outfile
]
class """ status update Replacement object for AVInfo provides
self.duration and self.video.r_frame_rate for encode()"""
def __init__(self, duration, rfr):
class Dict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
self.duration = duration
self.video = Dict({'r_frame_rate': rfr})
join_info = AVJoin(duration,rfr_list[0])
encode(ffmpeg, command, join_info)
def clear_markup(rec,infile,outfile):
logging.info('Started Clearing markup')
logging.debug('rec=%s infile=%s outfile=%s',rec,infile,outfile)
chanid = rec.chanid
utcstarttime = rec.starttime
starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', ''))
logging.debug('chanid=%s starttime=%s',chanid,starttime)
try:
rcl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearcutlist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output1 = rcl.communicate()
if rcl.wait() == 0:
logging.info('Cutlist removed for:%s_%s',chanid,starttime)
if rcl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing cutlist for:%s_%s',chanid,starttime)
logging.error('%s',output1)
except Exception as e:
logging.error('Mythutil exception clearing cutlist%s',e)
try:
rsl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearskiplist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rsl.communicate()[0]
if rsl.wait() == 0:
logging.info('Skiplist removed for:%s_%s',chanid,starttime)
if rsl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing skiplist for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythutil exception clearing skiplist:%s',e)
for index,mark in reversed(list(enumerate(rec.markup))):
if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END):
del rec.markup[index]
rec.bookmark = 0
rec.bookmarkupdate = datetime.now()
rec.cutlist = 0
rec.commflagged = 0
rec.markup.commit()
rec.basename = os.path.basename(infile)
rec.filesize = os.path.getsize(infile)
rec.transcoded = 1
rec.seek.clean()
rec.update()
try:
logging.info('Removing PNG files')
for png in glob('%s*.png' % infile):
os.remove(png)
except Exception as e:
logging.error('Error removing png files',e)
try:
logging.info('Removing JPG files')
for jpg in glob('%s*.jpg' % infile):
os.remove(jpg)
except Exception as e:
logging.error('Error removing jpg files',e)
try:
logging.info('Rebuilding seektable')
rst = subprocess.Popen(['mythcommflag','--chanid',str(chanid),'--starttime',str(starttime),'--rebuild'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Seektable Rebuilt for:%s_%s',chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Rebuilding Seektable for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythcommflag ERROR clearing skiplist:%s',e)
def tmp_chk(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.debug('chk:temp Folder found: %s',tmpdir)
if os.listdir(tmpdir) != 0:
logging.warning('chk:Temp folder not empty!:Removing Files: %s',tmpdir)
shutil.rmtree(tmpdir)
os.makedirs(tmpdir)
if not os.path.isdir(tmpdir):
logging.debug('chk:no temp folder found: %s',tmpdir)
os.makedirs(tmpdir)
logging.debug('chk:Temp folder created: %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def rem_tmp(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.debug('rem:temp Folder found %s',tmpdir)
shutil.rmtree(tmpdir)
if not os.path.isdir(tmpdir):
logging.debug('rem:temp Folder Removed %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def find_rec(chanid,starttime):
def local_time_offset(t=None):
if t is None:
t = time.time()
if time.localtime(t).tm_isdst and time.daylight:
return -time.altzone
else:
return -time.timezone
def RecordedFromBasename(chanid,starttime):
bnts = '%s_%s.ts' % (chanid,starttime)
bnmpg = '%s_%s.mpg' % (chanid,starttime)
x = list(db.searchRecorded(basename=bnmpg))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
x = list(db.searchRecorded(basename=bnts))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
raise LookupError('unable to find Recorded entry for '
'ChanID {} StartTime {}'
.format(chanid, starttime)
)
try:
rec = Recorded((chanid,starttime), db=db)
except:
try:
tzoffset = local_time_offset() / (60*60)
utcstarttime = datetime.strptime(starttime,"%Y%m%d%H%M%S")
utcstarttime = utcstarttime + timedelta(hours=tzoffset)
rec = Recorded((chanid,utcstarttime), db=db)
except:
rec = RecordedFromBasename(chanid,starttime)
return rec
def main():
parser = argparse.ArgumentParser(description='MythTV Commercial removal and closed caption extraction tool.\nNOTE:Having .srt file in same dir as media files breaks playback in some media players(VLC)')
parser.add_argument('--chanid',action='store',type=str,dest='chanid',help='Channel-Id of Recording')
parser.add_argument('--starttime',action='store',type=str,dest='starttime',help='Starttime of recording in utc format')
parser.add_argument('--jobid',action='store',type=int,dest='jobid',help='JOBID')
parser.add_argument('-o',action='store',type=str,dest='outfile',help='Output file to be created')
args = parser.parse_args()
if args.jobid:
run_cut(jobid=args.jobid,outfile=args.outfile)
sys.exit(0)
if args.chanid and args.starttime:
run_cut(chanid=args.chanid,starttime=args.starttime,outfile=args.outfile)
sys.exit(0)
else:
print('chanid and starttime or jobid required')
main()
Code: Select all
ffprobe -i file.ts -select_streams v -show_entries stream=r_frame_rate,avg_frame_rate
Code: Select all
#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
#MythTV commercial removal -- by Daraden
#usage userjob = /path to script/script.py --jobid %JOBID%
#/path to script/script.py --chanid --starttime can also be used
from __future__ import print_function
from MythTV import (Job, Program, Recorded, System, MythDB, findfile,
MythError, MythLog, datetime, VideoGrabber)
import argparse
import shutil
from glob import glob
import time
from datetime import timedelta
import subprocess
import ConfigParser
import logging
import logging.handlers
import sys
import os
import tempfile
# set this to True to use commflag results as cut-list
# this will automaticly flag commercials if no cut-list is available
use_commflag = True
# save copy of original file as file.old if not using output file
save_old = False
# user needs write access to this directory to save logfile
logdir ='/media/windows/test/'
LOG_FILENAME = '{}cut.log'.format(logdir)
# logging setup for file and console
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
lf= logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(lf)
logger.addHandler(ch)
try:
if os.access(logdir,os.W_OK):
fh = logging.handlers.TimedRotatingFileHandler(filename=LOG_FILENAME,
when='W0',interval=1,
backupCount=10
)
fh.setLevel(logging.DEBUG)
fh.setFormatter(lf)
logger.addHandler(fh)
if not os.access(logdir,os.W_OK):
logging.error('logdir not accessible:%s',logdir)
except exception as e:
logging.error('logdir not accessible:%s',logdir)
logging.error(e)
sys.exit(1)
try:
db=MythDB()
except exception as e:
logging.error(e)
sys.exit(1)
def prog_check():
"""checks for FFmpeg/FFprobe if not found MythFFmpeg/MythFFprobe
is used"""
from distutils import spawn
if spawn.find_executable('ffmpeg') != None:
ffmpeg = spawn.find_executable('ffmpeg')
elif (spawn.find_executable('ffmpeg') == None and
spawn.find_executable('mythffmpeg')):
ffmpeg = spawn.find_executable('mythffmpeg')
else:
ffmpeg = None
if spawn.find_executable('ffprobe') != None:
ffprobe = spawn.find_executable('ffprobe')
elif (spawn.find_executable('ffprobe') == None and
spawn.find_executable('mythffprobe')):
ffprobe = spawn.find_executable('mythffprobe')
else:
ffprobe = None
if ffmpeg != None and ffprobe != None:
return ffmpeg,ffprobe
else:
raise LookupError('Unable to find FFmpeg or MythFFmpeg')
#check for ffmpeg or mythffmpeg
ffmpeg = prog_check()[0]
logging.debug('ffmpeg={}'.format(ffmpeg))
ffprobe = prog_check()[1]
logging.debug('ffprobe={}'.format(ffprobe))
def run_cut(jobid=None,chanid=None,starttime=None,outfile=None):
logging.info('Started')
# Configure chanid and starttime from userjob input
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
starttime = job.starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
if not jobid:
chanid = chanid
starttime = starttime
logging.debug('chanid={} starttime={}'.format(chanid, starttime))
# Get database recording entry
rec = find_rec(chanid,starttime)
logging.debug('DB recording entry={}'.format(rec))
# Find and format full input file path
sg = findfile('/{}'.format(rec.basename), rec.storagegroup, db=db)
infile = os.path.join(sg.dirname, rec.basename)
# Assign and create temporary directory in recording directory
tmpdir = '{}{}{}/'.format(sg.dirname, 'crtmp.',
rec.basename.split('.')[0]
)
logging.debug('tmpdir is: {}'.format(tmpdir))
logging.info('Creating temporary directory')
tmp_chk(tmpdir)
# Get info from input file
avinfo = AVInfo(ffprobe,infile)
# Get video frame-rate
rfr = list(int(x) for x in avinfo.video.r_frame_rate.split('/'))
frame_rate = rfr[0] / float(rfr[1])
logging.debug('Video frame-rate is: {}'.format(frame_rate))
if save_old == True and outfile == None:
if jobid:
job.update({'status':job.RUNNING,
'comment':'Backing up original file'
}
)
logging.info('Backing up original file: %s',infile)
shutil.copyfile(infile,'%s.old' % infile)
logging.info('Finished backing up original file: {}'.format(infile))
# Configure output file
if outfile != None:
Outfile = outfile
elif outfile == None:
Outfile = infile
if rec.cutlist == 1 or use_commflag == True:
##Flag commercials##
if (use_commflag == True and rec.commflagged != 1
and rec.cutlist != 1
):
# need check for running commflag job?
try:
logging.info('Flagging commercials')
if jobid:
job.update({'status':job.RUNNING,
'comment': 'Flagging commercials'
}
)
startts = datetime.mythformat(rec.starttime)
rst = subprocess.Popen(['mythcommflag','--chanid',
str(chanid),'--starttime',
str(startts)],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Commercials flagged for:%s_%s',
chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Flagging '
'commercials for:{}_{}'
.format(chanid,starttime)
)
logging.error('MythCommflag allways exits with '
'decoding error?!'
)
# uncoment folowing line to log mythcommflag errors
#logging.error('{}'.format(output2))
except Exception as e:
logging.error('Mythcommflag ERROR: Flagging commercials %s',e)
# Reinitalize recording object
rec = find_rec(chanid,starttime)
if jobid:
job.update({'status':job.RUNNING,
'comment':'Getting Cutlist'
}
)
logging.info('getting cutlist')
# Get cut/skip list from database
if rec.cutlist == 1:
myth_cut_list = rec.markup.getcutlist()
if rec.cutlist != 1 and rec.commflagged == 1 and use_commflag:
myth_cut_list = rec.markup.getskiplist()
elif rec.cutlist == 0 and rec.commflagged != 1 and not use_commflag:
logging.debug('No cut/skip list found')
sys.exit(1)
logging.debug('Myth cut_list: {}'.format(myth_cut_list))
# Format cutlist, removing starting 0 and ending 9999999
cut_list = [mark for cuts in myth_cut_list for mark in cuts]
if cut_list[0] == 0:
cut_list.pop(0)
if cut_list[-1] == 9999999:
cut_list.pop(-1)
logging.debug('Cut list: {}'.format(cut_list))
# Get the name of the video codec
video_codec = avinfo.video.codec_name
logging.info('video codec: {}'.format(video_codec))
# Set cutlist string
if video_codec == 'h264':
cut_string = ','.join(str(i / frame_rate) for i in cut_list)
elif video_codec == 'mpeg2video':
cut_string = ','.join(str(i) for i in cut_list)
logging.debug('cut_string: {}'.format(cut_string))
# Create segment files
logging.info('Cutting')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Cutting started'
}
)
cut(ffmpeg, infile, cut_string, tmpdir)
# Create list of files to join
file_list = []
# Get list of segment files
for root, dirs, files in os.walk(tmpdir):
for File in files:
if (File.endswith('.ts') and
File.startswith('cut')):
if os.path.isfile(os.path.join(root,File)):
file_list.append(os.path.join(root,File))
# Set list of files to be joined
if myth_cut_list[0][0] == 0:
join_list = file_list[1::2]
if myth_cut_list[0][0] != 0:
join_list = file_list[0::2]
logging.debug('Join file list: {}'.format(join_list))
# Join segment files
logging.info('Joining segments')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Joining segments'
}
)
join(ffmpeg, join_list, outfile)
# Remove temporary directory
logging.info('removing temporary directory')
rem_tmp(tmpdir)
if outfile == None:
logging.info('clearing database markup')
if jobid:
job.update({'status':job.RUNNING,
'comment':'Clearing database markup'
}
)
clear_markup(rec,infile,outfile)
if jobid:
job.update({'status':job.FINISHED, 'comment':
'Commercial removal finished'
}
)
logging.info('Finished')
class DictToNamespace(object):
""" convert a dictonary and any nested dictonarys to namespace"""
def __setitem__(self, key, item):
self.__dict__[key] = item
def __getitem__(self, key):
return self.__dict__[key]
def has_key(self, k):
return self.__dict__.has_key(k)
def update(self, *args, **kwargs):
return self.__dict__.update(*args, **kwargs)
def keys(self):
return self.__dict__.keys()
def values(self):
return self.__dict__.values()
def items(self):
return self.__dict__.items()
def __contains__(self, item):
return item in self.__dict__
def __iter__(self):
return iter(self.__dict__)
def __init__(self, data):
for item,num in data.items():
if isinstance(num, str):
try:
num = float(num)
if float(num).is_integer():
num = int(num)
data.update({item: num})
except ValueError:
pass
self.__dict__.update(data)
for k, v in data.items():
if isinstance(v, dict):
self.__dict__[k] = DictToNamespace(v)
class AVInfo:
"""identify A/V configuration of input file and returns
self.video dict a list of self.audio.stream dicts,
and self.duration as float"""
def __init__(self=None,ffprobe=None,infile=None):
class Dict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
command = [ffprobe, '-v', '-8', '-show_entries',
'stream=codec_type,index,codec_name,channels,width,'
'height,r_frame_rate:stream_tags=language:'
'format=duration', '-of', 'csv=nk=0:p=0', infile
]
x = subprocess.check_output(command).decode('utf-8').split('\n')
vcd = {}
adict = {}
for vc in x:
if 'codec_type=video' in vc:
for item in vc.split(','):
k,v = item.split('=')
vcd.update({k:v})
for ac in x:
if 'codec_type=audio' in ac:
items =[item.split('=') for item in ac.split(',')]
streamdict ={k.strip(): v.strip() for (k,v) in items}
if 'tag:language' in streamdict.keys():
streamdict['language'] = (streamdict.pop
('tag:language')
)
adict.update({'stream{}'.format(streamdict['index'])
: streamdict})
for d in x:
if d.startswith('duration='):
dur = d.split('=')[1]
self.video = Dict(vcd)
self.audio = DictToNamespace(adict)
self.duration = dur
def encode(ffmpeg, command,AVInfo):
""" Run ffmpeg command with status output"""
# Length of progress bar
statlen = 10
# Charicter used for progress bar
# Use chr() in python 3
statchar = unichr(9619).encode('UTF-8')
# Charicter used to pad progress bar
pad = ' '
rfr = list(int(x) for x in AVInfo.video.r_frame_rate.split('/'))
frame_rate = float(rfr[0]) / float(rfr[1])
duration = float(AVInfo.duration)
total_frames = duration * frame_rate
with tempfile.TemporaryFile() as output:
process = subprocess.Popen(command, stdout=output,
stderr =output,
universal_newlines=True
)
while True:
if process.poll() is not None:
if process.poll() != 0:
output.seek(0)
print(output.read().decode('UTF-8'))
sys.exit(1)
if process.poll() == 0:
print('\rFinished{}'.format(pad * (statlen + 3)))
break
where = output.tell()
lines = output.read().decode('UTF-8')
if not lines:
time.sleep(0.1)
output.seek(where)
elif lines.startswith('frame='):
ln = ' '.join(lines.split()).replace('= ','=').split(' ')
for item in ln:
if item.startswith('frame='):
framenum = int(item.replace('frame=',''))
if item.startswith('fps='):
fps = float(item.replace('fps=',''))
if int(framenum) == 0:
pcomp = 0
else:
# python 2 div
pcomp = 100 * (float(framenum) / float(total_frames))
#python 3 div
#pcomp = 100 * (framenum / total_frames)
# python 2 div
stat = int((float(pcomp) / float(100)) * statlen)
# python 3 div
#stat = int((int(pcomp) / 100) * statlen)
padlen = statlen - stat
status = "|{:6.2f}%|".format(pcomp)
statusbar = '|{}{}|'.format(statchar * stat, pad * padlen)
status = '\r{}{}'.format(status,statusbar)
print(status, end="")
# Replace with flush=True in print function for python 3
sys.stdout.flush()
def cut(ffmpeg, infile, cutlist, tmpdir):
"""use ffmpeg segment muxer to cut infile into multipule files
using a csv formated cutlist placing them into tmpdir"""
cmd = [ffmpeg, '-ignore_unknown', '-i', infile, '-y', '-copyts',
'-start_at_zero','-c', 'copy','-map', '0'
]
av_check = AVInfo(ffprobe,infile)
for streams,stream in av_check.audio.items():
if stream.channels == '0':
cmd.extend(('-map',
('-0:{}'
.format(stream.index))))
if av_check.video.codec_name == 'mpeg2video':
cmd.extend(('-f','segment',
'-segment_frames', cutlist,
'{}cut%03d.ts'.format(tmpdir)
)
)
if av_check.video.codec_name == 'h264':
cmd.extend(('-f','segment',
'-segment_times', cutlist,
'{}cut%03d.ts'.format(tmpdir)
)
)
encode(ffmpeg, cmd, av_check)
def join(ffmpeg, file_list, outfile):
# Create FFmpeg concat string
concat_string = ','.join(file_list).replace(',','|')
duration_list = []
rfr_list = []
video_codec_list = []
for files in file_list:
file_avinfo = AVInfo(ffprobe,files)
# get file duration
duration_list.append(float(file_avinfo.duration.strip('\r')))
# Get file refrence frame rate
rfr_list.append(file_avinfo.video.r_frame_rate.strip('\r'))
# Get file video codec
video_codec_list.append(file_avinfo.video.codec_name.strip('\r'))
# Duration of joined files
duration = sum(duration_list)
# Check that all files refrence framerate are equal
rfr_match = all(rfr_list[0] == item for item in rfr_list)
if len(rfr_list) > 0 and rfr_match:
# Convert refrence frame rate to float
rfr = rfr_list[0].split('/')
frame_rate = int(rfr[0]) * float(rfr[1])
elif len(rfr_list) == 0 or not rfr_match:
raise ValueError('Incorrect or missing refrence frame rate')
# Estimate total number of frames
total_frames = duration * frame_rate
#check if all files video codec match
codec_match = all(video_codec_list[0] == item
for item in video_codec_list
)
if codec_match:
video_codec = video_codec_list[0]
if not codec_match:
raise ValueError('Not all video codecs match')
command = [ffmpeg, '-y', '-i', 'concat:{}'.format(concat_string),
'-map', '0', '-c', 'copy', '-f', 'mpegts', outfile
]
class AVJoin:
""" status update Replacement object for AVInfo provides
self.duration and self.video.r_frame_rate for encode()"""
def __init__(self, duration, rfr):
class Dict(dict):
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
self.duration = duration
self.video = Dict({'r_frame_rate': rfr})
join_info = AVJoin(duration,rfr_list[0])
encode(ffmpeg, command, join_info)
def clear_markup(rec,infile,outfile):
logging.info('Started Clearing markup')
logging.debug('rec=%s infile=%s outfile=%s',rec,infile,outfile)
chanid = rec.chanid
utcstarttime = rec.starttime
starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', ''))
logging.debug('chanid=%s starttime=%s',chanid,starttime)
try:
rcl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearcutlist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output1 = rcl.communicate()
if rcl.wait() == 0:
logging.info('Cutlist removed for:%s_%s',chanid,starttime)
if rcl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing cutlist for:%s_%s',chanid,starttime)
logging.error('%s',output1)
except Exception as e:
logging.error('Mythutil exception clearing cutlist%s',e)
try:
rsl = subprocess.Popen(['mythutil','--chanid',str(chanid),'--starttime',str(starttime),'--clearskiplist'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rsl.communicate()[0]
if rsl.wait() == 0:
logging.info('Skiplist removed for:%s_%s',chanid,starttime)
if rsl.wait() != 0:
logging.error('MYTHUTIL ERROR: clearing skiplist for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythutil exception clearing skiplist:%s',e)
for index,mark in reversed(list(enumerate(rec.markup))):
if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END):
del rec.markup[index]
rec.bookmark = 0
rec.bookmarkupdate = datetime.now()
rec.cutlist = 0
rec.commflagged = 0
rec.markup.commit()
rec.basename = os.path.basename(infile)
rec.filesize = os.path.getsize(infile)
rec.transcoded = 1
rec.seek.clean()
rec.update()
try:
logging.info('Removing PNG files')
for png in glob('%s*.png' % infile):
os.remove(png)
except Exception as e:
logging.error('Error removing png files',e)
try:
logging.info('Removing JPG files')
for jpg in glob('%s*.jpg' % infile):
os.remove(jpg)
except Exception as e:
logging.error('Error removing jpg files',e)
try:
logging.info('Rebuilding seektable')
rst = subprocess.Popen(['mythcommflag','--chanid',str(chanid),'--starttime',str(starttime),'--rebuild'], stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
output2 = rst.communicate()[0]
if rst.wait() == 0:
logging.info('Seektable Rebuilt for:%s_%s',chanid,starttime)
if rst.wait() != 0:
logging.error('MYTHcommflag ERROR: Rebuilding Seektable for:%s_%s',chanid,starttime)
logging.error('%s',output2)
except Exception as e:
logging.error('Mythcommflag ERROR clearing skiplist:%s',e)
def tmp_chk(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.debug('chk:temp Folder found: %s',tmpdir)
if os.listdir(tmpdir) != 0:
logging.warning('chk:Temp folder not empty!:Removing Files: %s',tmpdir)
shutil.rmtree(tmpdir)
os.makedirs(tmpdir)
if not os.path.isdir(tmpdir):
logging.debug('chk:no temp folder found: %s',tmpdir)
os.makedirs(tmpdir)
logging.debug('chk:Temp folder created: %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def rem_tmp(tmpdir):
try:
if os.path.isdir(tmpdir):
logging.debug('rem:temp Folder found %s',tmpdir)
shutil.rmtree(tmpdir)
if not os.path.isdir(tmpdir):
logging.debug('rem:temp Folder Removed %s',tmpdir)
except Exception as e:
logging.error('%s',e)
def find_rec(chanid,starttime):
def local_time_offset(t=None):
if t is None:
t = time.time()
if time.localtime(t).tm_isdst and time.daylight:
return -time.altzone
else:
return -time.timezone
def RecordedFromBasename(chanid,starttime):
bnts = '%s_%s.ts' % (chanid,starttime)
bnmpg = '%s_%s.mpg' % (chanid,starttime)
x = list(db.searchRecorded(basename=bnmpg))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
x = list(db.searchRecorded(basename=bnts))
if len(x) == 1:
for recorded in x:
return recorded
if len(x) != 1:
raise LookupError('unable to find Recorded entry for '
'ChanID {} StartTime {}'
.format(chanid, starttime)
)
try:
rec = Recorded((chanid,starttime), db=db)
except:
try:
tzoffset = local_time_offset() / (60*60)
utcstarttime = datetime.strptime(starttime,"%Y%m%d%H%M%S")
utcstarttime = utcstarttime + timedelta(hours=tzoffset)
rec = Recorded((chanid,utcstarttime), db=db)
except:
rec = RecordedFromBasename(chanid,starttime)
return rec
def main():
parser = argparse.ArgumentParser(description='MythTV Commercial removal and closed caption extraction tool.\nNOTE:Having .srt file in same dir as media files breaks playback in some media players(VLC)')
parser.add_argument('--chanid',action='store',type=str,dest='chanid',help='Channel-Id of Recording')
parser.add_argument('--starttime',action='store',type=str,dest='starttime',help='Starttime of recording in utc format')
parser.add_argument('--jobid',action='store',type=int,dest='jobid',help='JOBID')
parser.add_argument('-o',action='store',type=str,dest='outfile',help='Output file to be created')
args = parser.parse_args()
if args.jobid:
run_cut(jobid=args.jobid,outfile=args.outfile)
sys.exit(0)
if args.chanid and args.starttime:
run_cut(chanid=args.chanid,starttime=args.starttime,outfile=args.outfile)
sys.exit(0)
else:
print('chanid and starttime or jobid required')
main()