Skip to content

Instantly share code, notes, and snippets.

@tryone144

tryone144/bl_render.sh

Last active August 24, 2019 21:32
Show Gist options
  • Save tryone144/a5bbec510fd011c226d02f3cc862fd97 to your computer and use it in GitHub Desktop.
Save tryone144/a5bbec510fd011c226d02f3cc862fd97 to your computer and use it in GitHub Desktop.
Download ZIP
Launch multiple blender instances to speed up vse video encoding.
Raw
bl_render.sh
#!/bin/bash
#
# run as: ./render_final.sh BLENDFILE
#
# (c) 2019 Bernd Busse
#
BUILDROOT="$HOME/blender"
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
function print_usage() {
declare prog="$( basename "$1" )"
echo "usage: $prog [-v] [-y] [-c] [-o OUT] [-f CONTAINER[+VCODEC[+ACODEC]]] <BLENDFILE|ANIMATION>"
}
function print_help() {
cat <<xxEndOfHelpxx
Render animation of BLENDFILE using 'dist_render.py' to lossless intermediate.
Convert to final result with '-f' or '-o', or if ANIMATION is provided.
Parameters:
BLENDFILE Blender project file with the sequenced animation
ANIMATION Raw result of a previous run of the 'distributed_render.py' script
Options:
-h, --help Show this help message and exit
-v, --version Show version info and exit
-c, --nokeep Delete intermediate result after conversion
-y, --overwrite Overwrite existing files without asking
-f FMT, --format FMT convert intermediate according to FMT
-o OUT, --output OUT Safe result as OUT with container extension
FMT Specifier: CONTAINER[+VCODEC[+ACODEC]]
CONTAINER Container format ['mp4', 'mov', 'mkv']
VCODEC Video codec ['h264', 'h264_lossless', 'prores', 'prores_lossless', 'copy']
ACODEC Audio codec ['aac', 'mp3', 'copy']
CONTAINER Presets:
mp4: libx264[crf=20] + libfdk_aac[128k] in mp4
mp4_lossless: libx264[crf=0] + libfdk_aac[128k] in mp4
mov: libx264[crf=20] + libfdk_aac[128k] in mov (QuickTime)
lossless:
mov_lossless: prores_ks[copy] + copy in mov (QuickTime)
mkv: libx264[crf=20] + libfdk_aac[128k] in mkv (Matroska)
xxEndOfHelpxx
}
function fail() {
declare code="$1" && shift
declare msg="$@"
echo -e "\e[1;31mError:\e[0m $msg" >&2
if [[ "$code" -eq 3 ]]; then
print_usage "$0" >&2
fi
exit "$code"
}
function log_info() {
declare msg="$@"
echo -e "\e[1;34mInfo:\e[0m $msg"
}
function log_center() {
declare title=" [ ${1^^} ] "
declare -i count="$2"
count="${count:-64}"
declare -i padleft="$(( ("$count" + "${#title}") / 2 ))"
declare -i padright="$(( "$count" - "$padleft" ))"
declare pad="$( printf "%*s%*s\n" "$padleft" "${title// /_}" "$padright" "")"
pad="${pad// /=}"
echo -e "\e[1m${pad//_/ }\e[0m"
}
function parse_output_format() {
declare raw_format="$1"
declare container="$( echo "$raw_format" | cut -d '+' -s -f 1 )"
declare vcodec="$( echo "$raw_format" | cut -d '+' -s -f 2 )"
declare acodec="$( echo "$raw_format" | cut -d '+' -s -f 3 )"
declare ext
container="${container:-$raw_format}"
if [[ -z "$container" ]]; then
fail 3 "Missing format parameter for '-f'"
fi
# Check file container and set defaults
case "$container" in
"mp4")
vcodec="${vcodec:-h264}"
acodec="${acodec:-aac}"
ext="mp4"
;;
"mp4_lossless")
container="mp4"
vcodec="${vcodec:-h264_lossless}"
acodec="${acodec:-aac}"
ext="mp4"
;;
"mov")
vcodec="${vcodec:-h264}"
acodec="${acodec:-aac}"
ext="mov"
;;
"mov_lossless"|"lossless")
vcodec="${vcodec:-prores_lossless}"
acodec="${acodec:-copy}"
ext="mov"
;;
"mkv")
container="matroska"
vcodec="${vcodec:-h264}"
acodec="${acodec:-aac}"
ext="mkv"
;;
*)
fail 1 "Usupported container format '$container'" ;;
esac
# Check codec options
case "$vcodec" in
"h264")
vcodec="libx264" ;;
"h264_lossless")
vcodec="libx264:lossless" ;;
"prores")
vcodec="prores_ks" ;;
"prores_lossless"|"lossless")
vcodec="prores_ks:lossless" ;;
"copy") ;;
*)
fail 1 "Unsupported video codec '$vcodec'" ;;
esac
case "$acodec" in
"aac")
acodec="libfdk_aac" ;;
"mp3")
acodec="libmp3lame" ;;
"copy") ;;
*)
fail 1 "Unsupported audio codec '$acodec'" ;;
esac
echo "$container+$vcodec+$acodec+$ext"
}
function validate_inputfile() {
declare inputfile="$1"
if [[ -z "$inputfile" ]]; then
fail 3 "Missing mandatory paramter BLENDFILE|ANIMATION"
fi
if [[ ! -e "$inputfile" ]]; then
fail 1 "Cannot find BLENDFILE|ANIMATION '$inputfile'"
fi
inputfile="$( realpath "$inputfile" )"
if [[ ! -f "$inputfile" ]]; then
fail 1 "BLENDFILE|ANIMATION is not a file '$inputfile'"
fi
declare input_type
declare input_meta="$( file -b "$inputfile" )"
case "$input_meta" in
"Blender3D"*)
input_type="BLENDFILE" ;;
"Matroska data"*)
input_type="ANIMATION" ;;
*)
fail 1 "Unsupported type for '$inputfile': $input_meta" ;;
esac
echo "$input_type:$inputfile"
}
function dist_render() {
declare blendfile="$1"
declare buildroot="$2"
declare _render_script="$SCRIPTDIR/distributed_render.py"
if [[ ! -f "$_render_script" ]]; then
fail 1 "Cannot find render script '$_render_script'"
fi
declare _blender="$( which blender 2>/dev/null )"
if [[ -z "$_blender" ]] || [[ ! -x "$_blender" ]]; then
fail 1 "Cannot find blender executable in PATH"
fi
declare -a _command=("$_blender" -b "$blendfile" -P "$_render_script" -- --buildroot "$buildroot" --out "bl_render-raw")
log_info "Run command: ${_command[@]}"
log_center "start blender" 80
"${_command[@]}"
log_center "end blender" 80
}
function convert_ffmpeg() {
declare input="$1"
declare output="$2"
declare container="$3"
declare vcodec="$4"
declare acodec="$5"
declare _ffmpeg="$( which ffmpeg 2>/dev/null )"
if [[ -z "$_ffmpeg" ]] || [[ ! -x "$_ffmpeg" ]]; then
fail 1 "Cannot find ffmpeg executable in PATH"
fi
declare -a _video_options=("-c:v" "${vcodec%%:*}")
declare -a _audio_options=("-c:a" "${acodec%%:*}")
case "$vcodec" in
"libx264")
_video_options+=("-crf" "20" "-profile:v" "main" "-preset" "faster" "-pix_fmt" "yuv420p") ;;
"libx264:lossless")
_video_options+=("-crf" "0" "-profile:v" "main" "-preset" "veryslow" "-pix_fmt" "yuv420p") ;;
"prores_ks")
_video_options+=("-profile:v" "3" "-q:v" "11" "-pix_fmt" "yuv422p10le" "-vendor" "ap10") ;;
"prores_ks:lossless")
_video_options+=("-profile:v" "4" "-q:v" "0" "-vendor" "ap10") ;;
"copy") ;;
*)
fail 1 "Internal conversion error: No default parameters for vido-codec '$vcodec'" ;;
esac
case "$acodec" in
"libfdk_aac")
_audio_options+=("-b:a" "128k" "-ar" "48k") ;;
"libmp3lame")
_audio_options+=("-q:a" "0" "-ar" "48k") ;;
"copy") ;;
*)
fail 1 "Internal conversion error: No default parameters for audio-codec '$acodec'" ;;
esac
declare -a _command=("$_ffmpeg" -i "$input" "${_video_options[@]}" "${_audio_options[@]}" -f "$container" -y -- "$output")
log_info "Run command: ${_command[@]}"
log_center "start ffmpeg" 80
"${_command[@]}"
log_center "end ffmpeg" 80
}
function main() {
declare input input_type output
declare output_container output_vcodec output_acodec output_ext
declare flag_nokeep=false
declare flag_overwrite=false
# Parse commandline parameters
if [[ "$#" -lt 1 ]]; then
print_usage "$0" >&2
exit 1
fi
# Early parsing of '--help' and '--version'
for arg in "$@"; do
case "$arg" in
"-h"|"--help")
print_usage "$0"
print_help
exit 0 ;;
"-v"|"--version")
echo "bl_render.sh version 0.1"
exit 0 ;;
*) ;;
esac
done
# Parsing of all other arguments
while [[ "$#" -gt 0 ]]; do
case "$1" in
"-c"|"--nokeep")
flag_nokeep=true ;;
"-y"|"--overwrite")
flag_overwrite=true ;;
"-f"|"--format")
local fmt="$( parse_output_format "$2" )"
output_container="$( echo "$fmt" | cut -d '+' -f 1 )"
output_vcodec="$( echo "$fmt" | cut -d '+' -f 2 )"
output_acodec="$( echo "$fmt" | cut -d '+' -f 3 )"
output_ext="$( echo "$fmt" | cut -d '+' -f 4 )"
shift ;;
"-o"|"--output")
output="$2"
shift ;;
*)
if [[ -z "$input" ]]; then
local input_file="$( validate_inputfile "$1" )"
input="$( echo "$input_file" | cut -d ':' -f 2- )"
input_type="$( echo "$input_file" | cut -d ':' -f 1 )"
if [[ "$#" -gt 1 ]]; then
shift && fail 3 "Superfluous arguments: $@"
fi
else
fail 3 "Unkown argument: '$1'"
fi
;;
esac
shift
done
if [[ -z "$input" ]]; then
fail 1 "Missing mandatory paramter BLENDFILE|ANIMATION"
fi
# Build rendering arguments
echo -e "\e[1;32m == [ \e[31mBlender Render v0.1\e[32m ] == \e[0m"
declare project_name="$( basename "$input" )"
project_name="${project_name%%.*}"
declare project_root
declare raw_animation
if [[ "$input_type" = "BLENDFILE" ]]; then
project_root="$( realpath "$BUILDROOT/$project_name" )"
if [[ ! -d "$project_root" ]]; then
log_info "Create project root: $project_root"
mkdir -p -- "$project_root"
fi
raw_animation="$project_root/output/bl_render-raw.mkv"
if [[ -e "$raw_animation" ]]; then
echo -e "\e[1;33mWarning:\e[0m Raw output file '$raw_animation' already exists"
if [[ "$flag_overwrite" != true ]]; then
echo -ne " \e[1mOverwrite?\e[0m [y|N] "
read -re choice
case "$choice" in
"y"|"Y"|"j"|"J") ;;
"")
echo ;&
*)
fail 1 "Cannot render to '$raw_animation': File already exists" ;;
esac
fi
fi
# Render animation using render script
log_info "Render BLENDFILE: $input"
dist_render "$input" "$project_root"
if [[ "$?" -ne 0 ]]; then
fail 4 "Rendering of '$input' failed"
fi
if [[ ! -f "$raw_animation" ]]; then
fail 4 "Cannot find raw rendering of animation at: $raw_animation"
fi
log_info "Finished rendering of raw animation: $raw_animation"
elif [[ "$input_type" = "ANIMATION" ]]; then
raw_animation="$input"
project_root="$( cd "$( dirname "$input" )" >/dev/null 2>&1 && cd .. >/dev/null 2>&1 && pwd )"
log_info "Use pre-rendered animation: $raw_animation"
log_info "Use existing project root: $project_root"
output="${output:-bl_render-result}"
else
fail 1 "Internal error: Unhandled input type '$input_type'"
fi
# Gather output conversion arguments
if [[ -z "$output" ]] && [[ -z "$output_container" ]]; then
return
fi
output="${output:-bl_render-result}"
if [[ -z "$output_container" ]]; then
local fmt="$( parse_output_format "mp4" )"
output_container="$( echo "$fmt" | cut -d '+' -f 1 )"
output_vcodec="$( echo "$fmt" | cut -d '+' -f 2 )"
output_acodec="$( echo "$fmt" | cut -d '+' -f 3 )"
output_ext="$( echo "$fmt" | cut -d '+' -f 4 )"
fi
declare output_animation="$project_root/$output.$output_ext"
if [[ -e "$output_animation" ]]; then
echo -e "\e[1;33mWarning:\e[0m Output file '$output_animation' already exists"
if [[ "$flag_overwrite" != true ]]; then
echo -ne " \e[1mOverwrite?\e[0m [y|N] "
read -re choice
case "$choice" in
"y"|"Y"|"j"|"J") ;;
"")
echo ;&
*)
fail 1 "Cannot convert to '$output_animation': File already exists" ;;
esac
fi
fi
# Convert raw animation to requested format
log_info "Convert to '$output.$output_ext' as $output_container with $output_vcodec + $output_acodec"
convert_ffmpeg "$raw_animation" "$output_animation" "$output_container" "$output_vcodec" "$output_acodec"
if [[ ! -f "$output_animation" ]]; then
fail 4 "Cannot find conversion result at: $output_animation"
fi
log_info "Finished conversion to: $output_animation"
if [[ "$flag_nokeep" = true ]]; then
log_info "Cleanup intermediate results..."
rm -f -- "$raw_animation"
rmdir --ignore-fail-on-non-empty -- "$project_root/output"
fi
}
set -eo pipefail
# Run main
main "$@"
Raw
distributed_render.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# run as: blender -b BLENDFILE -P distributed_render.py
# outputs to 'dist_output.mkv' in lossless 'FFV1 + PCM' in Matroska container.
#
# (c) 2019 Bernd Busse
#
"""Blender script to export animation in lossess format using all cores.
Export blender animation in lossless FFV1 video- and PCM audio-codec.
Result is stored as 'dist-output_ISODATE.mkv' in Matroska container.
Run as: blender -p BLENDFILE -P distributed_render.py [--buildroot PATH]
The output will be placed in 'output/' next to BLENDFILE or in 'PATH/output/' if specified.
"""
import sys
import os
import datetime as dt
import subprocess
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
import bpy
from bpy import ops as op
from bpy import context as ctx
DIST_OUTPUT = "dist_output"
FINAL_OUTPUT = "output"
FRAME_COUNT = 250
WORKER_COUNT = 4
def eprint(*args, **kwargs):
"""Print to STDERR."""
return print("Error:", *args, file=sys.stderr, **kwargs)
def print_usage():
"""Print help message."""
print(__doc__, file=sys.stderr, end="")
def export_audio(dest):
"""Export complete sound."""
op.sound.mixdown('EXEC_DEFAULT',
filepath=os.path.join(dest, "audio.wav"),
check_existing=False,
container='WAV',
codec='PCM',
format='S16')
def render_frames(dest, start, end):
"""Export animation range (start, end]."""
# Get current scene and renderer
scene = ctx.scene
render = scene.render
# Set start and end frames
scene.frame_start = start
scene.frame_end = end
# Set video codec
render.image_settings.file_format = 'FFMPEG'
render.ffmpeg.format = 'MKV'
render.ffmpeg.codec = 'FFV1'
render.ffmpeg.audio_codec = 'NONE'
# Set output path
render.filepath = os.path.join(dest, "{:05d}-{:05d}.mkv".format(start, end))
# Render given frames
op.render.render(animation=True)
def main(blend_argv, argv):
"""Process `argv` and start export worker."""
# Parse '--buildroot' argument
buildroot = "//" # default to BLENDFILE path
if "--buildroot" in argv:
opt_idx = argv.index("--buildroot") + 1
if opt_idx >= len(argv):
raise RuntimeError("Missing argument for '--buildroot'")
buildroot = argv[opt_idx]
del argv[opt_idx]
del argv[opt_idx - 1]
# Parse '--out' argument
if "--out" in argv:
opt_idx = argv.index("--out") + 1
if opt_idx >= len(argv):
raise RuntimeError("Missing argument for '--out'")
outname = argv[opt_idx]
del argv[opt_idx]
del argv[opt_idx - 1]
buildroot = os.path.abspath(bpy.path.abspath(buildroot))
if not os.path.exists(buildroot):
raise RuntimeError("buildroot path does not exist: '{}'".format(buildroot))
elif not os.path.isdir(buildroot):
raise RuntimeError("buildroot is not a directory: '{}'".format(buildroot))
# Get blendfile path
output = os.path.join(buildroot, FINAL_OUTPUT)
dist_output = os.path.join(buildroot, DIST_OUTPUT)
if not os.path.exists(output):
os.mkdir(output)
if not os.path.exists(dist_output):
os.mkdir(dist_output)
script_args = ['--buildroot', buildroot]
# Handle different actions
if len(argv) == 0:
print(" == [ distributed_render.py v0.2 for blender ] == ")
print("Export to {}".format(buildroot))
# Get current scene
scene = ctx.scene
# Get start and end frame
scene.frame_start
scene.frame_end
dt_start = dt.datetime.now()
print("Start rendering of {} frames at {}"
.format(scene.frame_end - scene.frame_start + 1,
dt_start.time().isoformat()))
# Export sound
def export_audio_async(blender_cmd, args):
print("Export audio")
task_start = dt.datetime.now()
subprocess.run(blender_cmd + ['--', 'audio'] + args,
check=True, stdout=subprocess.DEVNULL)
task_end = dt.datetime.now()
print("Finshed exporting audio. Time elapsed: {}"
.format(str(task_end - task_start)))
# Render frames
def render_frames_async(blender_cmd, start, end, args):
print("Export frames " + start + " to " + end)
task_start = dt.datetime.now()
subprocess.run(blender_cmd + ['--', 'frames', start, end] + args,
check=True, stdout=subprocess.DEVNULL).check_returncode()
task_end = dt.datetime.now()
print("Finished exporting frames {} to {}. Time elapsed: {}"
.format(start, end, str(task_end - task_start)))
# Start pool execution
tasks = []
frames = []
with ThreadPoolExecutor(max_workers=WORKER_COUNT) as tpe:
task = tpe.submit(export_audio_async, blend_argv, script_args)
tasks.append(task)
for i in range(scene.frame_start,
scene.frame_end + 1,
FRAME_COUNT):
start_frame = i
end_frame = min(i + FRAME_COUNT - 1, scene.frame_end)
task = tpe.submit(render_frames_async, blend_argv,
str(start_frame), str(end_frame),
script_args)
tasks.append(task)
frames.append((start_frame, end_frame))
# Check return codes
for fut in futures.as_completed(tasks):
if fut.exception():
for task in tasks:
task.cancel()
raise fut.exception()
if outname:
filename = outname + ".mkv"
else:
filename = "dist_output_{:05d}-{:05d}.mkv".format(scene.frame_start,
scene.frame_end)
ffmpeg_args = ["ffmpeg", "-i", os.path.join(dist_output, "audio.wav"),
"-safe", "0", "-protocol_whitelist", "file,pipe",
"-f", "concat", "-i", "pipe:0",
"-c:v", "copy", "-c:a", "copy",
"-y", os.path.join(output, filename)]
ffmpeg = subprocess.Popen(ffmpeg_args,
stdin=subprocess.PIPE,
universal_newlines=False)
files = ["file '{}/{:05d}-{:05d}.mkv'\n"
.format(dist_output, start, end).encode('utf8')
for start, end in frames]
ffmpeg.stdin.writelines(files)
ffmpeg.stdin.flush()
ffmpeg.stdin.close()
retval = ffmpeg.wait()
if retval is not None and retval != 0:
eprint("merging output files with ffmpeg failed")
print("Cleanup intermediate files")
os.remove(os.path.join(dist_output, "audio.wav"))
for start, end in frames:
os.remove("{}/{:05d}-{:05d}.mkv".format(dist_output, start, end))
try:
os.rmdir(dist_output)
except OSError:
pass
dt_end = dt.datetime.now()
print("Finished rendering of {} frames at {}. Time elapsed: {}"
.format(scene.frame_end - scene.frame_start + 1,
dt_end.time().isoformat(),
str(dt_end - dt_start)))
elif argv[0] == 'audio':
# Export sound
print("Export audio")
export_audio(dist_output)
elif argv[0] == 'frames':
# Render given frames
print("Export frames " + argv[1] + " to " + argv[2])
render_frames(dist_output, int(argv[1]), int(argv[2]))
elif argv[0] in ('help', 'h'):
# Display help message
print_usage()
return
else:
raise RuntimeError("Unsupported action: " + str(argv))
if __name__ == '__main__':
try:
index = sys.argv.index('--')
except ValueError:
index = len(sys.argv)
try:
main(sys.argv[:index], sys.argv[index + 1:])
except RuntimeError as err:
eprint(str(err))
sys.exit(3)
except Exception as ex:
eprint("An unhandled Exception occured: " + str(ex))
sys.exit(1)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment