From 768e5e11cb2caebb96f1e6f48c0939a9588f9f7a Mon Sep 17 00:00:00 2001 From: "Yves G." Date: Mon, 8 Apr 2024 19:50:54 +0200 Subject: [PATCH] working version based on ffmpeg --- backup-disc.sh | 684 ++++++++++++++++++++++--------------------------- 1 file changed, 309 insertions(+), 375 deletions(-) diff --git a/backup-disc.sh b/backup-disc.sh index 7d1c67c..8e0f54a 100755 --- a/backup-disc.sh +++ b/backup-disc.sh @@ -6,13 +6,12 @@ # Dependencies (needed in PATH): # — isosize (part of util-linux) : get sector size and count of the disc (unless: -n) # — dd, ddrescue : copy image of disc (unless: -n) -# — HandBrakeCLI (part of HandBrake) : get disc metadata and rip chapters of each title to MKV -# — (mplayer) : detect black borders of each title -# — ffmpeg, ffprobe (part of FFMpeg) : extract single frames from an MKV, and manipulate streams +# — HandBrakeCLI (part of HandBrake) : get disc metadata +# — ffmpeg, ffprobe (part of FFMpeg) : extract single frames and rip to MKV # — spumux (part of dvdauthor) : generate missing VOBSUB streams to align all tracks # — identify, montage, convert (part of ImageMagick): get extracted frame metadata, create visual timelines # — czkawka_cli (part of Czkawka) : compare timelines and detect duplicate chapters -# — (mkvextract,) mkvmerge (part of MKVToolNix) : get MKV metadata and assemble chapters into a final MKV +# — mkvmerge (part of MKVToolNix) : get MKV metadata and assemble chapters into a final MKV # — sed, gawk, grep, sort, uniq, head, cut, tr, wc : data manipulation # — dirname, tee, ln, mktemp, cat, touch, mv, rm : file manipulation # — jq : for parsing JSON and extracting metadata @@ -37,27 +36,6 @@ # — https://www.reddit.com/r/mkvtoolnix/comments/11nbfy0/tutorial_mediumlinked_tiny_segmented_mkvs_with/ # — https://www.reddit.com/r/handbrake/comments/bhqxve/handbrake_settings_explained/ -# FIXME: HOW TO DETECT USELESS SUBTITLES? E.G. -# 3|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → Normal -# 5|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → ?? -# 6|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → Commentary -# ⇓ -# (about 4min for full movie for each subtitle track) -# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 3 --subtitle-burned=none -i .iso -o sub3.mkv -# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 5 --subtitle-burned=none -i .iso -o sub5.mkv -# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 6 --subtitle-burned=none -i .iso -o sub6.mkv -# (instantaneous) -# $ mkvextract sub3.mkv tracks --raw 1:sub3.sub -# $ mkvextract sub5.mkv tracks --raw 1:sub5.sub -# $ mkvextract sub6.mkv tracks --raw 1:sub6.sub -# $ ls -l sub* -# -rw-r--r-- 1 yves yves 287386078 8 mars 18:54 sub3.mkv -# -rw-r--r-- 1 yves yves 1058252 8 mars 19:35 sub3.sub -# -rw-r--r-- 1 yves yves 286304465 8 mars 19:07 sub5.mkv -# -rw-r--r-- 1 yves yves 2268 8 mars 19:35 sub5.sub -# -rw-r--r-- 1 yves yves 289029155 8 mars 19:12 sub6.mkv -# -rw-r--r-- 1 yves yves 2684952 8 mars 19:35 sub6.sub - trap 'exit 100' SIGINT # read command-line arguments @@ -84,7 +62,7 @@ KEEP_ALL=${BUILD:+true} BUILD="${BUILD:-$(mktemp -d /tmp/bdvdrip-XXXXXX)}" [ -d "$BUILD" ] && [ -w "$BUILD" ] || { echo "$BUILD is not a writeable directory"; exit 1; } DVD="${DVD:-/dev/sr0}" -[ -r "$DVD" ] && ! [ -d "$DVD" ] || [ -f "$BUILD/.iso" ] || { echo "$DVD is not a useable readable input"; exit 1; } +[ -r "$DVD" ] && ! [ -d "$DVD" ] || [ -e "$BUILD/.iso" ] || { echo "$DVD is not a useable readable input"; exit 1; } TARGET="${TARGET:-./output.mkv}" [ -d "$(dirname "$TARGET")" ] && [ -w "$(dirname "$TARGET")" ] || { echo "$(dirname "$TARGET") is not a writeable directory"; exit 1; } if ! [[ "$MAXW" =~ ^[0-9]+$ ]]; then MAXW=1280; fi @@ -92,7 +70,7 @@ if ! [[ "$MAXH" =~ ^[0-9]+$ ]]; then MAXH=720; fi exec 3>&2 function log_and_run() { - local getOutput + local getOutput output if [ "$1" == '-o' ]; then getOutput=1; shift fi @@ -101,16 +79,21 @@ function log_and_run() { printf '\n>> %s\n>> %s\n' "$step" "${*@Q}" >&3 fi case "$getOutput.$DEBUG" in - 1.debug) "$@" 1> >(tee >(cat >&3)) 2> >(tee >(cat >&3) >&2) ;; - 1.*) "$@" ;; - .debug) "$@" 1>&3 2>&3 ;; - .*) "$@" &>/dev/null ;; + 1.debug) "$@" 1> >(tee >(cat >&3)) 2> >(tee >(cat >&3) >&2) ;; #OK + 1.*) F=/tmp/fifo.$$.outerr; mkfifo $F; "$@" 1> >(tee -a $F) 2> >(tee -a $F >&2) & pid=$!; output="$(cat $F)"; rm -f $F; wait $pid ;; + .debug) "$@" 1>&3 2>&3 ;; #OK + .*) output="$("$@" 2>&1)" ;; esac + excode=$? + if [ $excode -ne 0 ] && [ "$DEBUG" != quiet ]; then + printf '%s>>> ERROR: EXIT CODE %s\n\n' "${output:+"$output"$'\n'}" "$excode" >&3 + fi + return $excode } # copy DVD -if [ ! -f "$BUILD/.iso" ]; then +if ! [ -f "$BUILD/.iso" ]; then if [ -n "$NO_DD" ]; then RM_ISO=${SHOW:+true} ln -s "$DVD" "$BUILD/.iso" @@ -118,7 +101,7 @@ if [ ! -f "$BUILD/.iso" ]; then read -r sectcount sectsize < <(LANG=C isosize -x "$DVD" | sed -r 's/.*: (.*), .*: (.*)$/\1 \2/') log_and_run 'Dump disc to ISO image' \ - dd -bs $sectsize count=$sectcount if="$DVD" of="$BUILD/.iso" || { + dd bs=$sectsize count=$sectcount if="$DVD" of="$BUILD/.iso" || { log_and_run 'Dump disc to ISO image (slower because of media errors)' \ ddrescue -n -b$sectsize "$DVD" "$BUILD/.iso" "$BUILD/.iso.mapfile" log_and_run 'Try to recover damaged sectors if needed' \ @@ -129,14 +112,14 @@ fi # fetch metadata -if [ ! -f "$BUILD/.json" ]; then +if ! [ -f "$BUILD/.json" ]; then log_and_run -o 'Read disc metadata (tracks, chapters…)' \ HandBrakeCLI --json -t 0 --min-duration 5 -i "$BUILD/.iso" 2>/dev/null \ | sed $'1i \\\n{\n1,/JSON Title Set:/d' >"$BUILD/.json" fi ALL_META="$( -# log_and_run -o 'Parse disc metadata' \ + log_and_run -o 'Parse metadata' \ jq -r ".TitleList[] | .Index as \$tnum | \ \"TITLE \(\$tnum) (\(.Duration.Hours):\(.Duration.Minutes):\(.Duration.Seconds))\", \ \"\tVideo (index, codec, geometry, frame-rate, bit-depth, chroma-subsampling, pixel aspect ratio, top:bottom:left:right borders, is interlaced)\", \ @@ -153,7 +136,7 @@ ALL_META="$( <"$BUILD/.json" )" -# show information then exit if requested +# show information then exit, if requested if [ -n "$SHOW" ]; then echo "$ALL_META" @@ -161,15 +144,23 @@ if [ -n "$SHOW" ]; then exit 0 fi -# parse information +# parse general information +declare TITLE_LIST=$(sed -rn 's/^TITLE (.*) .*/\1/p' <<<"$ALL_META") +declare VFORMAT=$(awk -F$'\t' '$5~"x480$"{print "ntsc";exit}' <<<"$ALL_META") declare -A VSECONDS +declare -A CHAPTERS declare -A CHNAMES +declare -A INTERLV while read -r tnum h m s; do VSECONDS[$tnum]=$((3600*h+60*m+s)); done < <( sed -nr 's/^TITLE (.*) \((.*):(.*):(.*)\)/\1 \2 \3 \4/p' <<<"$ALL_META") +for tnum in $TITLE_LIST; do CHAPTERS[$tnum]=$( + sed -rn "s/^C-$tnum\\t\\t([^\\t]*)\\t.*/\\1/p" <<<"$ALL_META"); done while read -r tnum chnum name; do CHNAMES[$tnum.$chnum]="$name"; done < <( sed -nr 's/^C-([0-9]+)\t\t([0-9]+)\t[0-9:]*\t(.*)/\1 \2 \3/p' <<<"$ALL_META") +while read -r tnum interl; do INTERLV[$tnum]="$interl"; done < <( + sed -nr 's/^V-([0-9]+)\t.*\t([^\t]*)$/\1 \2/p' <<<"$ALL_META") # compute final geometry @@ -186,10 +177,10 @@ function scale_round() { printf '%d %d %d %d %d %d' $adj1 $adj2 $dim1 $dim2 $5 $6 } -declare -A SRC_RAW SRC_LTRB SRC_WH2WH DAR_COUNT RIP_VIDEO +declare -A SRC_RAW SRC_LTRB SRC_WH2WH DAR_COUNT # → compute optimal crop and sizes, to target a pixel aspect ratio of 1:1, with width and height being multiples of 16 -for tnum in "${!VSECONDS[@]}"; do +for tnum in $TITLE_LIST; do read -r raww rawh parn pard cropt cropb cropl cropr < <(sed -nr " s/^V-$tnum\t(\t[^\t]*){2}\t([0-9]+)x([0-9]+)/\2 \3/; T s#(\t[^\t]*){3}\t([0-9]+)/([0-9]+)\t([0-9]+):([0-9]+):([0-9]+):([0-9]+)\t.*# \2 \3 \4 \5 \6 \7#p @@ -215,6 +206,8 @@ for tnum in "${!VSECONDS[@]}"; do fi fi fi + [ $srcw -le $raww ] || srcw=$raww + [ $srch -le $rawh ] || srch=$rawh SRC_LTRB[$tnum]="$cropl $cropt $cropr $cropb" SRC_WH2WH[$tnum]="$srcw $srch $finalw $finalh $scalen $scaled" SRC_RAW[$tnum]="$raww $rawh $parn $pard" @@ -227,7 +220,7 @@ read -r maxw maxh x < <( unset DAR_COUNT # → rescale each title according to the chosen width and height -for tnum in "${!VSECONDS[@]}"; do +for tnum in $TITLE_LIST; do read -r sw sh fw fh scn scd <<<"${SRC_WH2WH[$tnum]}" read -r rw rh parn pard <<<"${SRC_RAW[$tnum]}" # already OK ⇒ skip @@ -248,65 +241,14 @@ for tnum in "${!VSECONDS[@]}"; do fi SRC_WH2WH[$tnum]="$newsrcw $newsrch $neww $newh $newscalen $newscaled" done - -# → resize crop-box of titles when it can avoid double-encoding -for tnum in "${!VSECONDS[@]}"; do - read -r sw sh fw fh scn scd <<<"${SRC_WH2WH[$tnum]}" - read -r cl ct cr cb <<<"${SRC_LTRB[$tnum]}" - # already OK ⇒ skip - if [ $fw -eq $maxw ] && [ $fh -eq $maxh ]; then - RIP_VIDEO[$tnum]="--crop $ct:$cb:$cl:$cr --width $fw --height $fh --pixel-aspect 1:1 -e x265_10bit --encoder-profile main10 --vfr -q 18" - continue - fi - # else expand if it can reach the target geometry - read -r rw rh parn pard <<<"${SRC_RAW[$tnum]}" - if [ $parn -gt $pard ]; then - newsrcw=$((maxw*scd*pard/parn/scn)) - newsrch=$((maxh*scd/scn)) - maxraww=$((4*pard/parn+rw)) # same ¾-down/¼-up rounding rule - maxrawh=$((4+rh)) - else - newsrcw=$((maxw*scd/scn)) - newsrch=$((maxh*scd*parn/pard/scn)) - maxraww=$((4+rw)) # same ¾-down/¼-up rounding rule - maxrawh=$((4*parn/pard+rh)) - fi - if [ $newsrcw -le $maxraww ] && [ $newsrch -le $maxrawh ]; then - if [ $newsrcw -ge $maxraww ]; then - newcl=0; newcr=0; newsrcw=$rw; newsrch=$rh - else - newcl=$(( (rw-newsrcw)/2 )); newcr=$(( rw-newsrcw-newcl )) - fi - if [ $newsrch -ge $maxrawh ]; then - newct=0; newcb=0 - else - newct=$(( (rh-newsrch)/2 )); newcb=$(( rh-newsrch-newct )) - fi - SRC_WH2WH[$tnum]="$newsrcw $newsrch $maxw $maxh $scn $scd" - RIP_VIDEO[$tnum]="--crop $newct:$newcb:$newcl:$newcr --width $maxw --height $maxh --pixel-aspect 1:1 -e x265_10bit --encoder-profile main10 --vfr -q 18" - else - RIP_VIDEO[$tnum]="--crop $ct:$cb:$cl:$cr --width $fw --height $fh --pixel-aspect 1:1 -e x264_10bit --encoder-profile auto --vfr -q 15" - fi -done -unset SRC_LTRB unset SRC_RAW -## $1×$2, $3×$4: with×height of video, width×height of enclosing box -## &1: " " -#function compute_borders() { -# local top=$(( ($4-$2)/2 )) # ½Δ -# local left=$(( ($3-$1)/2 )) # ½Δ -# local bottom=$(( $4-$2-top )) -# local right=$(( $3-$1-left )) -# printf '%d %d %d %d' $left $top $right $bottom -#} - # compute final audio streams -declare -A SRC_AUDIO CATEG_COUNT RIP_AUDIO +declare -A SRC_AUDIO CATEG_COUNT # → categorize all audio streams -for tnum in "${!VSECONDS[@]}"; do +for tnum in $TITLE_LIST; do allaudio="$(awk -F$'\t' -vT=$tnum '$1=="A-" T{print T, $3, $5, $7, $8, $11, $4, $6, $10, $12}' <<<"$ALL_META")" # (technical info:) <1:title number> <2:stream number> <3:codec> <4:Hz> <5:bit/s> <6:secondary?> # (category:) <7:lang> <8:channels count> <9:commentary?> <10:visu.impaired?> <11:priority> @@ -318,8 +260,8 @@ for tnum in "${!VSECONDS[@]}"; do done )" while read -r x x cod freq bps x lng cnt cmt blind prio; do - k="$cod $freq $bps $lng $cnt $cmt $blind $prio" - CATEG_COUNT["$k"]=$(awk -vN=${CATEG_COUNT["$k"]:-0} "/C-$tnum\t/{N++};END{print N}" <<<"$ALL_META") + x="$cod $freq $bps $lng $cnt $cmt $blind $prio" + CATEG_COUNT["$x"]=$(awk -vN=${CATEG_COUNT["$x"]:-0} "/C-$tnum\t/{N++};END{print N}" <<<"$ALL_META") done <<<"${SRC_AUDIO[$tnum]}" done @@ -331,49 +273,39 @@ allaudio="$(export IFS=$'\n'; echo "${SRC_AUDIO[*]}")" MKV_AUDIO="$( for lng in ${ALL_LANGS//,/ }; do while read -r x x cod freq bps x lng cnt cmt blind prio; do - k="$cod $freq $bps $lng $cnt $cmt $blind $prio" + x="$cod $freq $bps $lng $cnt $cmt $blind $prio" name="${lng^^}${expo[$prio]} ${cnt}🕩" [ "$cmt" == true ] && name+=" 🗩" [ "$blind" == true ] && name+=" 🙈" - echo "${CATEG_COUNT["$k"]} $cod $freq $bps $lng $cnt $cmt $blind $prio $name" + echo "${CATEG_COUNT["$x"]} $cod $freq $bps $lng $cnt $cmt $blind $prio $name" done < <(awk -vL=$lng '$7==L{print}' <<<"$allaudio") \ | sort -k7,7 -k8,8 -k6,6nr -k9,9n -k1,1nr \ | uniq -f5 done \ | nl -nln -s' ' -w1 | cut -d' ' -f1,3- )" -unset expo +unset expo allaudio unset CATEG_COUNT -# → set audio ripping parameters to reach the target -declare -A count2mix=(["1"]=mono ["2"]=dpl2 ["3"]=dpl2 ["4"]=dpl2 ["5"]=dpl2 ["7"]=6point1 ["8"]=7point1) -for tnum in "${!VSECONDS[@]}"; do - declare -a nums=() cods=() mixs=() rates=() names=() - while read -r x anum cod freq bps x lng cnt cmt blind prio; do - read -r num targetc targetf targetr x x x x x name < <( - awk -vL=$lng -vC=$cnt -vM=$cmt -vB=$blind -vP=$prio '$5==L && $6==C && $7==M && $8==B && $9==P{print}' <<<"$MKV_AUDIO") - nums+=($anum) - rates+=(auto) - names+=("$num: ${name// / }") - if [[ $cod =~ ^(aac|ac3|eac3|truehd|dts|dtshd|mp2|mp3|flac|opus)$ ]]; then - cods+=("copy:$cod"); mixs+=("${count2mix[$cnt]:-5point1}") - else - cods+=("flac16"); mixs+=("${count2mix[$cnt]:-5point1}") - fi - done <<<"${SRC_AUDIO[$tnum]}" - RIP_AUDIO["$tnum"]="$(export IFS=, - printf -- '-a %s -E %s --mixdown %s -R %s -A %s' \ - "${nums[*]:-none}" "${cods[*]:-none}" "${mixs[*]:-none}" "${rates[*]:-none}" "${names[*]:-none}" - )" - unset nums cods mixs rates names -done +# (technical info:) <1:stream number> <2:MKV codec> <3:FFMpeg index> <4:FFMpeg codec> <5:FFMpeg layout> <6:Hz> <7:bit/s> +# (category:) <8:lang> <9:channels count> <10:commentary?> <11:visu.impaired?> <12:priority> <13:name> +MKV_AUDIO="$( + while read -r num cod freq bps lng count cmt blind prio name; do + # take example on an existing stream + read -r ffnum t2 < <(export IFS=$'\n'; echo "${SRC_AUDIO[*]}" \ + | awk '$1!=T{T=$1;C=0};{print ++C, $0}' | sed -nr "s|^([^ ]+) ([^ ]+) [^ ]+ $cod $freq $bps [^ ]+ $lng $count $cmt $blind $prio\$|\\1 \\2|p;T;q") + read -r ffcod ffchanlay < <(ffprobe -hide_banner -output_format json -show_streams -select_streams a -f dvdvideo -title $t2 "$BUILD/.iso" 2>/dev/null \ + | jq -r --argjson I $ffnum '.streams[] | select(.index == $I) | "\(.codec_name) \(.channel_layout)"') + echo "$num $cod $ffnum $ffcod $ffchanlay $freq $bps $lng $count $cmt $blind $prio $name" + done <<<"$MKV_AUDIO" +)" # compute final subtitle streams -declare -A SRC_SUB RIP_SUB +declare -A SRC_SUB # → categorize all subtitle streams -for tnum in "${!VSECONDS[@]}"; do +for tnum in $TITLE_LIST; do allsubs="$(awk -F$'\t' -vT=$tnum '$1=="S-" T{print $3, $8, $5, $4, $7, $9}' <<<"$ALL_META")" # (technical info:) <1:stream number> <2:forced?> # (category:) <3:codec> <4:lang> <5:commentary?> <6:hear.impaired?> <7:priority> @@ -404,55 +336,9 @@ MKV_SUB="$( )" unset expo -# → set subtitle ripping parameters to reach the target -declare -A count2mix=(["1"]=mono ["2"]=dpl2 ["3"]=dpl2 ["4"]=dpl2 ["5"]=dpl2 ["7"]=6point1 ["8"]=7point1) -for tnum in "${!VSECONDS[@]}"; do - declare -a nums=() names=() - while read -r snum x cod lng cmt deaf prio; do - name="$(sed -nr "s/^([^ ]+) $cod $lng $cmt $deaf $prio /\1: /p" <<<"$MKV_SUB")" - nums+=($snum) - names+=("${name// / }") - done <<<"${SRC_SUB[$tnum]}" - RIP_SUB["$tnum"]="$(export IFS=,; printf -- '-s %s -S %s' "${nums[*]:-none}" "${names[*]:-none}")" - unset nums names -done - -# rip chapters - -#2024-03-20 19:01 yrc: with ffmpeg DVD demuxer you can extract the chapters in one shot now if that helps -#2024-03-20 19:01 ffmpeg -f dvdvideo -chapter_start 5 -chapter_end 5 -i DVD_INPUT -map 0 -c copy Chapter5.mkv -#2024-03-20 19:02 it will also apply dispositions for commentary, karaoke, AD audio if dvd had tagged them correctly - -if [ -z "$(ls "$BUILD/"tmp.*.ch.*.mkv 2>/dev/null)" ]; then - for chapter in "${!CHNAMES[@]}"; do - tnum=${chapter%.*} - chnum=${chapter#*.} - - log_and_run "Rip track $tnum chapter $chnum (${CHNAMES[$chapter]})" \ - HandBrakeCLI -t $tnum --min-duration 5 -c $chnum --no-markers \ - ${RIP_VIDEO[$tnum]} ${RIP_AUDIO[$tnum]} ${RIP_SUB["$tnum"]} --subtitle-burned=none \ - --non-anamorphic --no-comb-detect --no-decomb --no-detelecine \ - --no-hqdn3d --no-nlmeans --no-chroma-smooth --no-unsharp --no-lapsharp --no-deblock \ - -i "$BUILD/.iso" -o "$BUILD/tmp.$tnum.ch.$chnum.mkv" <2:MKV codec> <3:FFMpeg index> <4:FFMpeg codec> <5:FFMpeg layout> <6:Hz> <7:bit/s> -# (category:) <8:lang> <9:channels count> <10:commentary?> <11:visu.impaired?> <12:priority> <13:name> -MKV_AUDIO="$( - while read -r num cod freq bps lng count cmt blind prio name; do - # take example on an existing stream - read -r ffnum t2 < <(export IFS=$'\n'; echo "${SRC_AUDIO[*]}" \ - | awk '$1!=T{T=$1;C=0};{print ++C, $0}' | sed -nr "s|^([^ ]+) ([^ ]+) [^ ]+ $cod $freq $bps [^ ]+ $lng $count $cmt $blind $prio\$|\\1 \\2|p;T;q") - read -r ffcod ffchanlay < <(ffprobe -hide_banner -select_streams a -output_format json -show_streams "$BUILD/tmp.$t2.ch.1.mkv" 2>/dev/null \ - | jq -r --argjson I $ffnum '.streams[] | select(.index == $I) | "\(.codec_name) \(.channel_layout)"') - echo "$num $cod $ffnum $ffcod $ffchanlay $freq $bps $lng $count $cmt $blind $prio $name" - done <<<"$MKV_AUDIO" -)" - # generate per-chapter timeline snapshots -if [ -z "$(ls "$BUILD/"tmp.*.ch.*.png 2>/dev/null)" ]; then +if [ -z "$(ls "$BUILD/"t.*.ch.*.png 2>/dev/null)" ]; then [ -d "$BUILD/.tsnap" ] || mkdir "$BUILD/.tsnap" while read -r tnum chnum x; do seconds=$(awk -F$'\t' -vT=$tnum -vC=$chnum '$3==C && $1=="C-" T{FS=":"; $0=$4; print 3600*$1+60*$2+$3}' <<<"$ALL_META") @@ -464,7 +350,8 @@ if [ -z "$(ls "$BUILD/"tmp.*.ch.*.png 2>/dev/null)" ]; then while [ $snapsec -le $seconds ]; do target="$BUILD/.tsnap/t.$tnum.ch.$chnum.s.$snapsec.png" log_and_run "Make snapshot @${snapsec}s of track $tnum chapter $chnum" \ - ffmpeg -hide_banner -i "$BUILD/tmp.$tnum.ch.$chnum.mkv" -ss ${snapsec}s -vf scale=w=256:h=144:force_original_aspect_ratio=decrease -vframes 1 "$target%1d.png" /dev/null)" ]; then log_and_run "Annotate timeline snapshot of track $tnum chapter $chnum with audio and subtitles metadata" \ convert -fill red -gravity SouthEast -annotate +0+0 \ "$(export IFS=+; echo "${audio[*]}")|$(export IFS=+; echo "${subs[*]}")" \ - "$BUILD/.tsnap/t.$tnum.ch.$chnum.png" "$BUILD/tmp.$tnum.ch.$chnum.png" "$BUILD/.duplicates" fi -# fill-in missing streams - -if [ -z "$(ls "$BUILD/"t.*.ch.*.mkv 2>/dev/null)" ]; then - for tnum in "${!VSECONDS[@]}"; do - read -r x x tw th x <<<"${SRC_WH2WH[$tnum]}" - while read -r chnum chtimeceil; do - declare -A blank=() - ffcmd=(ffmpeg -hide_banner -i "$BUILD/tmp.$tnum.ch.$chnum.mkv") - ffmap=(-map_chapters -1 -map 0:V:0) - if [ $tw -eq $maxw ] && [ $th -eq $maxh ]; then - ffenc=(-c:v:0 copy -copyinkf:v:0 -copyts:v:0) - else - # https://superuser.com/a/991412 - ffenc=(-c:v:0 hevc -hwaccel:v:0 auto -profile:v:0 main10 -fps_mode:v:0 vfr -qscale:v:0 18 -preset:v:0 slower -pix_fmt:v:0 + -enc_time_base:v:0 demux -vf:v:0 "scale=w=${maxw}:h=${maxh}:force_original_aspect_ratio=1,pad=${maxw}:${maxh}:(ow-iw)/2:(oh-ih)/2") - fi - - # audio streams - while read -r mkvnum mkvcod ffnum ffcod ffchanlay mkvfreq mkvrate lng count cmt blind prio name; do - read -r x num cod freq rate x < <( - awk -vL=$lng -vC=$count -vM=$cmt -vB=$blind -vP=$prio '$7==L && $8==C && $9==M && $10==B && $11==P{print}' <<<"${SRC_AUDIO[$tnum]}") - if [ -z "$num" ]; then - # no such stream in this title - f="$BUILD/tmp.empty_a.$ffchanlay.$mkvfreq.$mkvrate.$chtimeceil.$ffcod" - if ! [ -f "$f" ]; then - log_and_run "For title $tnum chapter $chnum, create missing audio stream for ${chtimeceil}s of ${mkvnum}: ${name}" \ - ffmpeg -hide_banner -lavfi anullsrc=channel_layout="${ffchanlay}":sample_rate=${mkvfreq}:duration=${chtimeceil} -c:a ${ffcod} -b:a ${mkvrate} "$f" &2 - exit 1 - else - # no such stream in this title - f="$BUILD/tmp.empty_s.$chtimeceil.mpeg2" - if ! [ -f "$f" ]; then - if ! [ -f "$BUILD/tmp.empty_pixel.png" ]; then - log_and_run "Generate a transparent pixel for use as an empty subtitle stream" \ - convert -size 1x1 'xc:rgba(0,0,0,0)' "$BUILD/tmp.empty_pixel.png" - fi - log_and_run -o "Generate XML description for ${chtimeceil}s of empty subtitle" \ - echo "" >"$BUILD/tmp.empty_s.xml" - log_and_run -o "For title $tnum chapter $chnum, create missing subtitle stream for ${chtimeceil}s of ${mkvnum}: ${name}" \ - spumux -m dvd --nomux --nodvdauthor-data "$BUILD/tmp.empty_s.xml" "$f" - ffcmd+=(-i "$f") - blank["$f"]=$((${#blank[*]}+1)) - fi - ffmap+=(-map ${blank["$f"]}:s:0) - ffenc+=(-c:s:$((mkvnum-1)) copy) - fi - done <<<"$MKV_SUB" - - log_and_run "Adapt title $tnum chapter $chnum to final MKV streams" \ - "${ffcmd[@]}" "${ffmap[@]}" "${ffenc[@]}" -shortest "$BUILD/t.$tnum.ch.$chnum.mkv" - unset blank ffcmd ffmap ffenc - rm -f "$BUILD/tmp.empty_s.xml" "$BUILD/tmp.empty_pixel.png" - done < <(awk -F$'\t' -vT=$tnum '$1=="C-" T{bFS=FS; n=$3; FS=":"; $0=$4; print n, 3600*$1+60*$2+$3+1; FS=bFS}' <<<"$ALL_META") - done -fi - -exit - -# fetch chapter metadata - -if [ -z "$(ls "$BUILD/"t.*.ch.*.json 2>/dev/null)" ]; then - for mkv in "$BUILD/"t.*.ch.*.mkv; do - log_and_run "Read metadata (length, codecs…) for $mkv" \ - mkvmerge -F json -i "$mkv" -r "${mkv%mkv}json" - done -fi - -declare -A STREAMS_IDS -declare -A CHAPTERS - -jqscript='[' -jqscript+='([.tracks[] | [select(.type=="video").properties] | sort_by(.number)[] | "\(.number),\(.codec_id),\(.pixel_dimensions)"] | join("+"))' -jqscript+=',([.tracks[] | [select(.type=="audio").properties] | sort_by(.number)[] | "\(.number),\(.codec_id),\(.audio_channels)x\(.audio_sampling_frequency),\(.language)"] | join("+"))' -jqscript+=',([.tracks[] | [select(.type=="subtitles").properties] | sort_by(.number)[] | "\(.number),\(.codec_id),\(.language)"] | join("+"))' -jqscript+='] | join("|")' -TITLE_LIST='' -while read -r tnum chnum; do - TITLE_LIST="${TITLE_LIST% $tnum} $tnum" - CHAPTERS[$tnum]="${CHAPTERS[$tnum]} $chnum" -done < <(ls -1 "$BUILD/"t.*.ch.*.json | sed -r 's#.*/t\.(.*)\.ch\.(.*)\.json#\1 \2#' | sort -k1,1n -k2,2n) -for tnum in $TITLE_LIST; do - read chnum x <<<"${CHAPTERS[$tnum]}" - STREAMS_IDS[$tnum]="$(jq -r "$jqscript" <"$BUILD/t.$tnum.ch.$chnum.json")" -done - -# detect source tracks that will fit together within the same end-result track - -if [ ! -f "$BUILD/.titlelist" ]; then - for tnum in $TITLE_LIST; do - strid="${STREAMS_IDS[$tnum]}" - lowestwithid=$(for i in "${!STREAMS_IDS[@]}"; do printf '%d\t%s\t\n' $i "${STREAMS_IDS[$i]}"; done \ - | grep -F $'\t'"$strid"$'\t' | sort -t$'\n' -k1,1n | head -n 1 | cut -f1) - printf '%d %d %d %d\n' $(islongfilm $tnum) $(isepisode $tnum) $lowestwithid $tnum - done \ - | sort -k1,1n -k2,2n -k3,3n -k4,4n | awk ' - $1=="0"{print $4; next} - ($2 $3)!=streams{if(streams!=""){printf("\n")};streams=($2 $3);printf("%d",$4);next} - {printf(" %d",$4)} - END{printf("\n")} - ' >"$BUILD/.titlelist" -fi - -# compute chapters’ timestamps, UIDs, and substitutions + prepare merging - -declare -A TIMESTAMPS -declare -a STREAMS_LIST -declare -A STREAMS_TO_REF -declare -A STREAMS_TO_FILES -declare -A CHAPTERS_IDS -#declare -A UIDS -declare -A SUBST -declare -a MKVMRGARGS - while read -r t1 ch1 t2 ch2; do SUBST[$t2.$ch2]="$t1.$ch1" done <"$BUILD/.duplicates" -while read -r tlist; do - strid="${STREAMS_IDS[${tlist%% *}]}" - if printf '%s\n' "${STREAMS_LIST[@]}" | grep -qxF "$strid"; then - ref=${STREAMS_TO_REF[$strid]} - else - ref=0 - STREAMS_LIST+=("$strid") - fi - for tnum in $tlist; do - for chnum in ${CHAPTERS[$tnum]}; do - tch=$tnum.$chnum - while [ -n "${SUBST[$tch]}" ]; do tch=${SUBST[$tch]}; done - if [ $tch == $tnum.$chnum ]; then - # compute milliseconds - start=$ref - stop=$(bc -lq <<<"$ref+$(jq -r '.container.properties | .duration/.timestamp_scale' <"$BUILD/t.$tnum.ch.$chnum.json")") - ref=$stop - TIMESTAMPS[$tch]="$start $stop" - #UIDS[$tch]="$(jq -r .container.properties.segment_uid <"$BUILD/t.$tnum.ch.$chnum.json")" - STREAMS_TO_FILES[$strid]="${STREAMS_TO_FILES[$strid]}|$BUILD/t.$tnum.ch.$chnum.mkv" - else - TIMESTAMPS[$tnum.$chnum]="${TIMESTAMPS[$tch]}" - #UIDS[$tnum.$chnum]="${UIDS[$tch]}" - fi - CHAPTERS_IDS[$tnum.$chnum]="$(jq -r .container.properties.segment_uid <"$BUILD/t.$tnum.ch.$chnum.json" \ - | tr '[a-f]' '[A-F]' | bc -q <<<"ibase=16;$(cat)" | head -c 10)" - done - done - STREAMS_TO_REF[$strid]=$ref -done <"$BUILD/.titlelist" +# rip chapters and fill-in missing streams -declare -A STREAMS_TO_OFFSET +declare -A USED_A USED_S + +for tnum in $TITLE_LIST; do + read -r sw sh fw fh scn scd <<<"${SRC_WH2WH[$tnum]}" + read -r cropl cropt cropr cropb <<<"${SRC_LTRB[$tnum]}" + firstch=true + while read -r chnum chtimefloor; do + [ -z "${SUBST[$tnum.$chnum]}" ] || continue + declare -A blank=() + + # video stream + ffcmd=( + ffmpeg -hide_banner -f dvdvideo -preindex 1 -trim false + -title $tnum -chapter_start $chnum -chapter_end $chnum -i "$BUILD/.iso" + ) + ffmap=(-map_chapters -1 -map 0:V:0) + ffenc=() + filter= + if [ "${INTERLV[$tnum]}" == true ]; then + filter+=",bwdif" + fi + filter+=",crop=x=$cropl:y=$cropt:w=$((sw/2*2)):h=$((sh/2*2)):exact=1" + if [ $sw -ne $fw ] || [ $sh -ne $fh ]; then + ffenc+=(-sws_flags lanczos+accurate_rnd) + filter+=",scale=w=$fw:h=$fh:force_original_aspect_ratio=disable,setsar=1/1" + fi + if [ $fw -ne $maxw ] || [ $fh -ne $maxh ]; then + filter+=",pad=w=$maxw:h=$maxh:x=-1:y=-1" + fi + ffenc+=(-filter:v:0 "${filter:1}" -enc_time_base:v:0 demux) + ffenc+=(-c:v:0 libx265 -x265-params:v:0 profile=main10:preset=slower:crf=18:sar=1:videoformat=${VFORMAT:-pal}:rc-lookahead=120:bframes=12:ref=6:subme=7:aq-mode=3) + + # audio streams + while read -r mkvnum mkvcod ffnum ffcod ffchanlay mkvfreq mkvrate lng count cmt blind prio name; do if [ -n "$mkvnum" ]; then + read -r x num cod freq rate x < <( + awk -vL=$lng -vC=$count -vM=$cmt -vB=$blind -vP=$prio '$7==L && $8==C && $9==M && $10==B && $11==P{print}' <<<"${SRC_AUDIO[$tnum]}") + if [ -z "$num" ]; then + # no such stream in this title + f="$BUILD/tmp.empty_a.$ffchanlay.$mkvfreq.$mkvrate.$chtimefloor.$ffcod" + if ! [ -f "$f" ] && ! [ -f "$BUILD/t.$tnum.ch.$chnum.mkv" ]; then + log_and_run "For title $tnum chapter $chnum, create missing audio stream for ${chtimefloor}s of ${mkvnum}: ${name}" \ + ffmpeg -hide_banner -strict -2 -lavfi anullsrc=channel_layout="${ffchanlay}":sample_rate=${mkvfreq}:duration=${chtimefloor} -c:a ${ffcod} -b:a ${mkvrate} "$f" &2 + exit 1 + else + # no such stream in this title + f="$BUILD/tmp.empty_s.${VFORMAT:-pal}.$chtimefloor.mpeg2" + if ! [ -f "$f" ] && ! [ -f "$BUILD/t.$tnum.ch.$chnum.mkv" ]; then + if ! [ -f "$BUILD/tmp.empty_pixel.png" ]; then + log_and_run "Generate a transparent pixel for use as an empty subtitle stream" \ + convert -size 1x1 'xc:rgba(0,0,0,0)' "$BUILD/tmp.empty_pixel.png" + fi + log_and_run -o "Generate XML description for ${chtimefloor}s of empty subtitle" \ + echo "" >"$BUILD/tmp.empty_s.xml" + log_and_run -o "For title $tnum chapter $chnum, create missing subtitle stream for ${chtimefloor}s of ${mkvnum}: ${name}" \ + spumux -m dvd --nomux --nodvdauthor-data "$BUILD/tmp.empty_s.xml" "$f" + fi + if [ -z "${blank["$f"]}" ]; then + ffcmd+=(-i "$f") + blank["$f"]=$((${#blank[*]}+1)) + fi + ffmap+=(-map ${blank["$f"]}:s:0) + ffenc+=(-c:s:$((mkvnum-1)) copy) + fi + ffenc+=(-metadata:s:s:$((mkvnum-1)) language=$lng -metadata:s:s:$((mkvnum-1)) title="$mkvnum: $name") + fi; done <<<"$MKV_SUB" + firstch= + + if ! [ -f "$BUILD/t.$tnum.ch.$chnum.mkv" ]; then + log_and_run "Adapt title $tnum chapter $chnum to final MKV streams" \ + "${ffcmd[@]}" "${ffmap[@]}" "${ffenc[@]}" "$BUILD/t.$tnum.ch.$chnum.mkv" "$BUILD/.titlelist" + unset tcategs +fi + +# fetch chapter metadata + +for mkv in "$BUILD/"t.*.ch.*.mkv; do + if ! [ -f "${mkv%mkv}json" ]; then + log_and_run "Read metadata (length, ids…) for $mkv" \ + mkvmerge -F json -i "$mkv" -r "${mkv%mkv}json" + fi +done + +# compute chapters’ timestamps, UIDs, and substitutions + prepare merging + +declare -A TIMESTAMPS +declare -A CHAPTERS_IDS +declare -A UIDS +declare -a MKVMRGARGS=() + +default= +strnum=0 +while read -r x x x x x x x lng x cmt blind x; do if [ -n "$lng" ]; then + strnum=$((strnum+1)) + if [ "$default$cmt$blind${lng,,}" == "falsefalse${DEFAULT_LANG,,}" ]; then + default=found + MKVMRGARGS+=(--default-track-flag $strnum:1) + else + MKVMRGARGS+=(--default-track-flag $strnum:0) + fi +fi; done <<<"$MKV_AUDIO" +while read -r x x lng cmt deaf x; do if [ -n "$lng" ]; then + strnum=$((strnum+1)) + if [ "$default$cmt$blind" == "falsefalse" ] && grep -qiF ",${lng,,}," <<<",$ALL_LANGS,"; then + default=found + MKVMRGARGS+=(--default-track-flag $strnum:1) + else + MKVMRGARGS+=(--default-track-flag $strnum:0) + fi +fi; done <<<"$MKV_SUB" +unset default strnum ref=0 -for strid in "${STREAMS_LIST[@]}"; do - STREAMS_TO_OFFSET[$strid]=$ref - file="${STREAMS_TO_FILES[$strid]##*|}" - tnum=$(sed -r 's#.*/t.([0-9]+).ch.[0-9]+.mkv#\1#' <<<"$file") - streamcount=$(jq -r '.tracks | length' <"${file%mkv}json") - for ((i=0; i @@ -733,9 +648,7 @@ ENDOFXML edflagdefault=1 titlenum=1 - while read -r titlelist; do - strid="${STREAMS_IDS[${titlelist%% *}]}" - offset=${STREAMS_TO_OFFSET[$strid]} + while IFS=$'\t' read -r label titlelist; do cat < @@ -743,24 +656,24 @@ ENDOFXML $edflagdefault 1 - $(printf '%03d' $titlenum): $titlelist ($(tr '|+' ' ' <<<"$strid")) + $(printf '%03d %s' $titlenum "$label") ENDOFXML chflaghidden=0 for tnum in $titlelist; do - for chnum in ${CHAPTERS[$tnum]}; do + for chnum in ${CHAPTERS[$tnum]}; do if [ -f "$BUILD/t.$tnum.ch.$chnum.mkv" ]; then cat < ${CHAPTERS_IDS[$tnum.$chnum]} - $(formatTimestamp $(printf '%s+%s\n' $offset ${TIMESTAMPS[$tnum.$chnum]% *} | bc -q)) - $(formatTimestamp $(printf '%s+%s\n' $offset ${TIMESTAMPS[$tnum.$chnum]#* } | bc -q)) + $(formatTimestamp ${TIMESTAMPS[$tnum.$chnum]% *}) + $(formatTimestamp ${TIMESTAMPS[$tnum.$chnum]#* }) $chflaghidden 1 - + - $tnum.$chnum — ${CHNAMES[$tnum.$chnum]} + $tnum.$chnum (A${USED_A[$tnum]:- ∅} - S${USED_S[$tnum]:- ∅}) — ${CHNAMES[$tnum.$chnum]} ENDOFXML @@ -768,7 +681,7 @@ ENDOFXML if [ "${titlelist/ }" != "$titlelist" ]; then chflaghidden=1 fi - done + fi; done chflaghidden=0 done @@ -789,7 +702,7 @@ fi # merge all chapters of all titles -if [ ! -f "$TARGET" ]; then +if ! [ -f "$TARGET" ]; then log_and_run 'Build final MKV file' \ mkvmerge --disable-track-statistics-tags --append-mode file --chapters "$BUILD/mkv-editions.xml" -o "$TARGET" "${MKVMRGARGS[@]}" fi @@ -797,3 +710,24 @@ fi if [ $? -eq 0 ] && [ -z "$KEEP_ALL" ]; then rm -rf "$BUILD" fi + +# FIXME: HOW TO DETECT USELESS SUBTITLES? E.G. +# 3|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → Normal +# 5|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → ?? +# 6|fra,VOBSUB:bitmap,false,false,false Francais (Wide Screen) [VOBSUB] → Commentary +# ⇓ +# (about 4min for full movie for each subtitle track) +# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 3 --subtitle-burned=none -i .iso -o sub3.mkv +# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 5 --subtitle-burned=none -i .iso -o sub5.mkv +# $ HandBrakeCLI -t 1 -e x265_10bit --encoder-profile main10 -q 1 --vfr -X 128 -Y 96 -a none -s 6 --subtitle-burned=none -i .iso -o sub6.mkv +# (instantaneous) +# $ mkvextract sub3.mkv tracks --raw 1:sub3.sub +# $ mkvextract sub5.mkv tracks --raw 1:sub5.sub +# $ mkvextract sub6.mkv tracks --raw 1:sub6.sub +# $ ls -l sub* +# -rw-r--r-- 1 yves yves 287386078 8 mars 18:54 sub3.mkv +# -rw-r--r-- 1 yves yves 1058252 8 mars 19:35 sub3.sub +# -rw-r--r-- 1 yves yves 286304465 8 mars 19:07 sub5.mkv +# -rw-r--r-- 1 yves yves 2268 8 mars 19:35 sub5.sub +# -rw-r--r-- 1 yves yves 289029155 8 mars 19:12 sub6.mkv +# -rw-r--r-- 1 yves yves 2684952 8 mars 19:35 sub6.sub