Last active
April 6, 2022 06:10
-
-
Save jakeogh/dd44d2f998b7c6eac9e1985f3db9fc59 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # License: The Unlicense | |
| # This simulates a user launching yt-dlp to download playlist A: | |
| # "https://www.youtube.com/playlist?list=PLaWNxqesaSyhjP0duSOH8C_MJROI1tmD6" | |
| # and then later deciding to launch a second yt-dlp process to download a _different_ playlist B: | |
| # "https://www.youtube.com/playlist?list=PLmlhzhpHN7Z2e5DPni8bWuNr9_OVR2nCX" | |
| # | |
| # The problem is, playlist A and B both contain ID Gwo3pEH7hUE. | |
| # | |
| # This duplicate ID causes a race condition (depending on the specifics of the playlist length and | |
| # download options), where one process can encounter Gwo3pEH7hUE while the other process already | |
| # has an advisory lock on the video or audio track. | |
| # | |
| # The race shouldnt matter, because the process without a lock should not write to the file, but it does, because: | |
| # utils.py: | |
| # self.f = io.open(filename, mode, encoding=encoding) | |
| # results in: | |
| # strace: | |
| # openat(AT_FDCWD</delme/_yt_dlp_delme/_yt_dlp_test_1646631804>, "youtube__Gwo3pEH7hUE.f160.mp4.part", O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC, 0666) = 4</delme/_yt_dlp_delme/_yt_dlp_test_1646631804/youtube__Gwo3pEH7hUE.f160.mp4.part> | |
| # | |
| # Which truncates the file, even though this process does not have an advisory lock. | |
| # | |
| # The below test is using --match-filter and --size to make the race condition trigger reliabely. | |
| log(){ | |
| echo -e "$*" > /dev/stderr | |
| } | |
| log "\n\nSTARTING $0" "$@" | |
| yt_dlp_HEAD_repo="/home/user/_myapps/yt-dlp" # this repo's HEAD should reflect the version of yt-dlp this script exectutes | |
| test -d "${yt_dlp_HEAD_repo}/.git/" || { echo "plese set yt_dlp_HEAD_repo on line 33 of $(readlink -f "$0")" ; exit 1 ; } | |
| cd "${yt_dlp_HEAD_repo}" || exit 1 | |
| git_HEAD_commit=$(git rev-parse --short HEAD) | |
| cd - | |
| log "git_HEAD_commit: ${git_HEAD_commit}" | |
| boolish(){ | |
| test "${1}" -eq 0 && echo "PASS" && exit 0 | |
| test "${1}" -eq 1 && echo "fail" && exit 0 | |
| } | |
| ffmpeg_test_file(){ | |
| #file=$(readlink -f "${1}") # too verbose | |
| file="${1}" | |
| log "\nffmpeg_test_file(): ${file}" | |
| #pwd | |
| test -e "${file}" || { log "\nCODE_ERROR: file: ${file} does not exist\n" ; exit 1 ; } | |
| ffmpeg -v error -xerror -i "${file}" -f null - 2>&1 | |
| ffmpeg_exit_code_test_1="$?" | |
| log "ffmpeg_exit_code_test_1: ${ffmpeg_exit_code_test_1}" | |
| # this exits > 0 IFF ffmpeg encountered an error (by detecting any bytes it writes to stdout) | |
| if read -d '' -rn1 < <(set -m ; exec unbuffer ffmpeg -threads 2 -v error -i "${file}" -f null /dev/stdout); then printf 'ffmpeg: %s detected an error\n' "$!"; kill -TERM $!; fi; wait "$!" | |
| ffmpeg_exit_code_test_2="$?" | |
| if [ "${ffmpeg_exit_code_test_1}" -gt 0 ] && [ "${ffmpeg_exit_code_test_2}" -gt 0 ]; | |
| then | |
| log "\nERROR: both ffmpeg test methods agree: ${file} is corrupt:\n" | |
| log "ftrace log for ${file}:" | |
| grep ${file} fatrace.log || exit 1 | |
| log | |
| return "${ffmpeg_exit_code_test_1}" | |
| elif [ "${ffmpeg_exit_code_test_1}" -eq 0 ] && [ "${ffmpeg_exit_code_test_2}" -eq 0 ]; | |
| then | |
| log "both ffmpeg test methods agree: ${1} is good" | |
| return "${ffmpeg_exit_code_test_2}" | |
| else | |
| log "\nCODE_ERROR: ffmpeg tests disagree" | |
| exit 1 | |
| fi | |
| } | |
| whereis fatrace > /dev/null || { log "fatrace not found, it's used to capture the file operations, please install" ; exit 1; } | |
| whereis ffmpeg > /dev/null || { log "ffmpeg not found, is used to detect corrupt files, please install" ; exit 1; } | |
| whereis stdbuf > /dev/null || { log "stdbuf not found, is used to make correctly ordered log files, please install" ; exit 1; } | |
| whereis strace > /dev/null || { log "strace not found, is used to log system calls, please install" ; exit 1; } | |
| sudo /bin/true # prompt user for sudo pw so the fatrace line below executes immediately | |
| # make stdout unbuffered so logging looks nice (not sure it's working) | |
| # https://stackoverflow.com/a/30975598 | |
| # https://stackoverflow.com/questions/30975334/copy-unbuffered-stdout-to-file-from-within-bash-script-itself | |
| if [[ "$1" != __UNBUFFERED__ ]]; then | |
| #set -x | |
| stdbuf -oL "$0" __UNBUFFERED__ | |
| #set +x | |
| else | |
| shift #discard __UNBUFFERED__ | |
| set -o nounset | |
| temp_dir="_yt_dlp_test_$(date +%s)___HEAD:${git_HEAD_commit}" | |
| mkdir "${temp_dir}" || exit 1 | |
| cd "${temp_dir}" || exit 1 | |
| # capture this terminal session to logfile | |
| exec > >(tee -i terminal_log.txt) | |
| exec 2>&1 | |
| log "Using temp_dir: ${temp_dir}" | |
| cp -v "$0" . # add a copy of this script to the log folder | |
| sudo fatrace --current-mount --timestamp --timestamp --output fatrace.log & | |
| sudo_pid="$!" | |
| sleep 2 # make sure fatrace is active | |
| fatrace_pid=$(ps --ppid "${sudo_pid}" -o pid=) | |
| #log "fatrace_pid: ${fatrace_pid}" | |
| set -x | |
| # process_A (playlist has 70 videos) (the filter pairs it down to 3, one being Gwo3pEH7hUE) | |
| unbuffer strace -ff -y -s 128 --output yt-dlp.strace.log --timestamps=format:unix,precision:ns \ | |
| /usr/bin/yt-dlp "https://www.youtube.com/playlist?list=PLaWNxqesaSyhjP0duSOH8C_MJROI1tmD6" \ | |
| --match-filter "id~='\b[A-Z].*[A-Z]E\b'" \ | |
| -S +size \ | |
| --output "%(extractor)s__%(id)s.%(ext)s" \ | |
| --keep-video \ | |
| --no-color --verbose > process_A.yt_dlp.log 2>&1 & | |
| set +x | |
| process_A_pid="$!" | |
| log "process_A ${process_A_pid} started" | |
| ps -f -auxww | grep "${process_A_pid}" | |
| set -x | |
| # process_B (playlist has 34 videos) (the filter pairs the list down to 1, Gwo3pEH7hUE) | |
| unbuffer strace -ff -y -s 128 --output yt-dlp.strace.log --timestamps=format:unix,precision:ns \ | |
| /usr/bin/yt-dlp "https://www.youtube.com/playlist?list=PLmlhzhpHN7Z2e5DPni8bWuNr9_OVR2nCX" \ | |
| --match-filter "id~='\b[A-Z].*[A-Z]E\b'" \ | |
| -S +size \ | |
| --output "%(extractor)s__%(id)s.%(ext)s" \ | |
| --keep-video \ | |
| --no-color --verbose > process_B.yt_dlp.log 2>&1 & | |
| set +x | |
| process_B_pid="$!" | |
| log "process_B ${process_B_pid} started" | |
| ps -f -auxww | grep "${process_A_pid}" | |
| log "waiting for processes: ${process_A_pid} ${process_B_pid} to exit" | |
| wait "${process_A_pid}" ; log "process_A exited $?" | |
| wait "${process_B_pid}" ; log "process_B exited $?" | |
| #pwd | |
| #ls -alh | |
| sudo kill "${fatrace_pid}" || exit 1 | |
| cat fatrace.log | grep -v strace > fatrace.log.no_strace.log || exit 1 | |
| mv -f fatrace.log.no_strace.log fatrace.log || exit 1 | |
| sed -i "s/^/[${process_A_pid}] /" process_A.yt_dlp.log || exit 1 | |
| sed -i "s/^/[${process_B_pid}] /" process_B.yt_dlp.log || exit 1 | |
| # make strace-log-merge include the fatrace log | |
| cp -i fatrace.log yt-dlp.strace.log."${fatrace_pid}" || exit 1 | |
| strace-log-merge yt-dlp.strace.log > yt-dlp.strace.log || exit 1 | |
| test -s yt-dlp.strace.log && rm -f yt-dlp.strace.log.* || exit 1 | |
| log "INFO: unlinking unrelated ID's, any failures here are unexpected" | |
| set -x | |
| rm -f youtube__Q6NBnPfPhWE.f139.m4a || exit 1 | |
| rm -f youtube__Q6NBnPfPhWE.f160.mp4 || exit 1 | |
| rm -f youtube__Q6NBnPfPhWE.mp4 || exit 1 | |
| rm -f youtube__V1aONINVkSE.f139.m4a || exit 1 | |
| rm -f youtube__V1aONINVkSE.f160.mp4 || exit 1 | |
| rm -f youtube__V1aONINVkSE.mp4 || exit 1 | |
| set +x | |
| expected_output_file_count=3 | |
| ls -alh | |
| output_file_count="$(find . -type f -name "youtube__Gwo3pEH7hUE*" | wc -l)" | |
| if [ "${output_file_count}" -ne "${expected_output_file_count}" ]; | |
| then | |
| log "\nERROR: (${expected_output_file_count}) youtube__* output files were expected, but (${output_file_count}) were found:\n" | |
| fi | |
| /bin/ls -alh || exit 1 | |
| if [ -s youtube__Gwo3pEH7hUE.f160.mp4 ] && [ -s youtube__Gwo3pEH7hUE.f139.m4a ] && [ ! -e youtube__Gwo3pEH7hUE.mp4 ]; | |
| then | |
| log "\nERROR: both youtube__Gwo3pEH7hUE.f160.mp4 and youtube__Gwo3pEH7hUE.f139.m4a exist, but youtube__Gwo3pEH7hUE.mp4 does not, regardless, the manual merge attempt below will attempt to check if either input file is corrupt\n" | |
| fi | |
| set -x | |
| ffmpeg -y -loglevel warning -xerror -i file:youtube__Gwo3pEH7hUE.f160.mp4 -i file:youtube__Gwo3pEH7hUE.f139.m4a -c copy -map 0:v:0 -map 1:a:0 -movflags +faststart file:youtube__Gwo3pEH7hUE.manual.mp4 | |
| ffmpeg_manual_merge_exit_code="$?" | |
| set +x | |
| #log "ffmpeg_manual_merge_exit_code: ${ffmpeg_manual_merge_exit_code}" | |
| if [ "${ffmpeg_manual_merge_exit_code}" -gt 0 ]; | |
| then | |
| log "\nERROR: ffmpeg manual merge job exited nonzero, either one or both of the input files is corrupt" | |
| if [ "${output_file_count}" -eq ${expected_output_file_count} ]; | |
| then | |
| #echo "NotImplementedError(automerge worked but manual merge failed?)" ; exit 1 | |
| echo "ERROR: the automerge worked, but the manual merge failed. Need a logging filesystem to understand what happened without parsing strace logs, but most likely, one of the input files changed after the automerge read them." | |
| test -s youtube__Gwo3pEH7hUE.mp4 || { echo "IMPLEMENTATION_ERROR: youtube__Gwo3pEH7hUE.mp4 exists" ; exit 1 ; } | |
| test ! -x youtube__Gwo3pEH7hUE.manual.mp4 || { echo "IMPLEMENTATION_ERROR: youtube__Gwo3pEH7hUE.manual.mp4 does not exist" ; exit 1 ; } | |
| fi | |
| else | |
| log "\nINFO: ffmpeg manual merge job exited 0, it appears that there is no file corruption, checking each input file again anyway.\n" | |
| if [ "${output_file_count}" -eq 2 ]; | |
| then | |
| log "ERROR... but neither process_A or process_B tried to merge, so this is the 'nobody merged, no corruption' case.\n" | |
| fi | |
| fi | |
| log "\nINFO: checking each Gwo3pEH7hUE input file:" | |
| log "\nls -alh youtube__Gwo3pEH7hUE*" | |
| ls -alh youtube__Gwo3pEH7hUE* || exit 1 | |
| ffmpeg_test_file youtube__Gwo3pEH7hUE.f160.mp4 | |
| video_test_exit_code=$? | |
| log "video_test_exit_code: ${video_test_exit_code}" | |
| ffmpeg_test_file youtube__Gwo3pEH7hUE.f139.m4a | |
| audio_test_exit_code=$? | |
| log "audio_test_exit_code: ${audio_test_exit_code}" | |
| # whew ba/sh | |
| total="Unknown" | |
| merge_attempted="Unknown" | |
| grep -E "ffmpeg command line.*Gwo3pEH7hUE" process_A.yt_dlp.log || grep -E "ffmpeg command line.*Gwo3pEH7hUE" process_B.yt_dlp.log || merge_attempted="False" && merge_attempted="True" | |
| if [ "${output_file_count}" -eq 2 ]; | |
| then | |
| if [ "${merge_attempted}" = "True" ]; | |
| then | |
| automerge="fail" | |
| elif [ "${merge_attempted}" = "False" ]; | |
| then | |
| automerge="None" | |
| fi | |
| elif [ "${output_file_count}" -eq ${expected_output_file_count} ]; | |
| then | |
| if [ "${merge_attempted}" = "True" ]; | |
| then | |
| automerge="PASS" | |
| elif [ "${merge_attempted}" = "False" ]; | |
| then | |
| echo "NotImplementedError(3 files, but no merge attempted?)"; exit 1 | |
| fi | |
| else | |
| echo "NotImplementedError"; exit 1 | |
| fi | |
| if [ "${output_file_count}" -eq 3 ]; | |
| then | |
| ffmpeg_test_file youtube__Gwo3pEH7hUE.mp4 | |
| automerged_file_ffmpeg_test_exit_code="$?" | |
| if [ "${automerged_file_ffmpeg_test_exit_code}" -eq 0 ]; | |
| then | |
| total="PASS" | |
| fi | |
| else | |
| total="fail" | |
| fi | |
| result_string="VIDEO:$(boolish ${video_test_exit_code})__AUDIO:$(boolish ${audio_test_exit_code})__AUTOMERGE:${automerge}__MANUALMERGE:$(boolish ${ffmpeg_manual_merge_exit_code})__TOTAL:${total}" | |
| #log "\nresult_string: ${result_string}" | |
| result_folder="${temp_dir}___${result_string}" | |
| cd .. | |
| mv -vi "${temp_dir}" "${result_folder}" | |
| log "\nINFO: results written to: $(readlink -f "${result_folder}")" | |
| fi | |
| # Typical results of running this script multiple times: | |
| # IMPORTANT Notes: | |
| # 1. for evaluation of the O_TRUNC problem, only the MANUALMERGE col matters | |
| # 2. the 3 MANUALMERGE:fail instances with PR #2294 applied are due to a different race condition | |
| # With PR #2294 to fix O_TRUNC: | |
| # Notes: | |
| # the "AUTOMAEGE:fail" lines are a different (set of) issues, unrelated to O_TRUNC. They are races, timing dependent, I'll send another PR on that. | |
| # the (1x) "VIDEO:fail" and (2x "AUDIO:fail") (causing the 3x MANUALMERGE:fail) are also unrelated to O_TRUNC, and needs it's own PR (also a race) | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 16:47:40 _yt_dlp_test_1646783120___HEAD:4010dcbe5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 16:52:03 _yt_dlp_test_1646783338___HEAD:4010dcbe5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 16:56:38 _yt_dlp_test_1646783643___HEAD:4010dcbe5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 16:59:48 _yt_dlp_test_1646783862___HEAD:202454028___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 17:07:46 _yt_dlp_test_1646784064___HEAD:202454028___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:12:57 _yt_dlp_test_1646784582___HEAD:aa994ac8f___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:17:39 _yt_dlp_test_1646784837___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:22:27 _yt_dlp_test_1646785140___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 17:27:42 _yt_dlp_test_1646785388___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:32:16 _yt_dlp_test_1646785717___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:37:53 _yt_dlp_test_1646786042___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 17:45:00 _yt_dlp_test_1646786407___HEAD:342cd60f5___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 17:52:14 _yt_dlp_test_1646786724___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 17:59:10 _yt_dlp_test_1646787319___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 18:08:04 _yt_dlp_test_1646787642___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:17:44 _yt_dlp_test_1646788360___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:22:59 _yt_dlp_test_1646788732___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 18:25:30 _yt_dlp_test_1646789011___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:28:01 _yt_dlp_test_1646789181___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:30:41 _yt_dlp_test_1646789297___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:32:50 _yt_dlp_test_1646789461___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:34:43 _yt_dlp_test_1646789590___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:36:37 _yt_dlp_test_1646789700___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 18:38:48 _yt_dlp_test_1646789812___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:40:55 _yt_dlp_test_1646789956___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:43:14 _yt_dlp_test_1646790076___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:45:24 _yt_dlp_test_1646790214___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:47:32 _yt_dlp_test_1646790347___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:49:26 _yt_dlp_test_1646790469___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 18:51:36 _yt_dlp_test_1646790584___HEAD:342cd60f5___VIDEO:PASS__AUDIO:fail__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:53:32 _yt_dlp_test_1646790706___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 18:55:38 _yt_dlp_test_1646790834___HEAD:342cd60f5___VIDEO:PASS__AUDIO:fail__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:57:28 _yt_dlp_test_1646790949___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 18:59:33 _yt_dlp_test_1646791067___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 19:01:26 _yt_dlp_test_1646791188___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 19:03:32 _yt_dlp_test_1646791307___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 19:05:38 _yt_dlp_test_1646791427___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 19:07:33 _yt_dlp_test_1646791556___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 19:09:44 _yt_dlp_test_1646791667___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 19:11:51 _yt_dlp_test_1646791807___HEAD:342cd60f5___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # Without PR: | |
| # | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 20:54:52 _yt_dlp_test_1646798004___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 20:56:23 _yt_dlp_test_1646798099___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 20:57:59 _yt_dlp_test_1646798188___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 240 2022-03-08 20:59:40 _yt_dlp_test_1646798286___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:01:37 _yt_dlp_test_1646798412___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:03:14 _yt_dlp_test_1646798503___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:04:59 _yt_dlp_test_1646798611___HEAD:e491d06d3___VIDEO:PASS__AUDIO:fail__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:06:45 _yt_dlp_test_1646798711___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:08:18 _yt_dlp_test_1646798810___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:09:53 _yt_dlp_test_1646798905___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:11:43 _yt_dlp_test_1646799008___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:13:09 _yt_dlp_test_1646799108___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:14:43 _yt_dlp_test_1646799196___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:16:12 _yt_dlp_test_1646799295___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:17:38 _yt_dlp_test_1646799378___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:19:13 _yt_dlp_test_1646799464___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:20:47 _yt_dlp_test_1646799559___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:22:19 _yt_dlp_test_1646799654___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:23:49 _yt_dlp_test_1646799744___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:25:10 _yt_dlp_test_1646799835___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:26:37 _yt_dlp_test_1646799916___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:28:05 _yt_dlp_test_1646800003___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:29:33 _yt_dlp_test_1646800092___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:31:01 _yt_dlp_test_1646800178___HEAD:e491d06d3___VIDEO:fail__AUDIO:fail__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:32:26 _yt_dlp_test_1646800263___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:33:56 _yt_dlp_test_1646800351___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:35:38 _yt_dlp_test_1646800449___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:37:09 _yt_dlp_test_1646800553___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:38:35 _yt_dlp_test_1646800635___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:40:19 _yt_dlp_test_1646800728___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 220 2022-03-08 21:41:48 _yt_dlp_test_1646800826___HEAD:e491d06d3___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:43:20 _yt_dlp_test_1646800923___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:44:45 _yt_dlp_test_1646801005___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:46:19 _yt_dlp_test_1646801091___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:47:48 _yt_dlp_test_1646801184___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:49:19 _yt_dlp_test_1646801274___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:50:40 _yt_dlp_test_1646801364___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:52:08 _yt_dlp_test_1646801445___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:53:32 _yt_dlp_test_1646801534___HEAD:e491d06d3___VIDEO:fail__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # 0 drwxr-xr-x 2 user user 200 2022-03-08 21:54:54 _yt_dlp_test_1646801618___HEAD:e491d06d3___VIDEO:PASS__AUDIO:fail__AUTOMERGE:fail__MANUALMERGE:fail__TOTAL:fail | |
| # typical folder contents: | |
| # $ ls _yt_dlp_test_1649191011___HEAD:554800c85___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:PASS__MANUALMERGE:PASS__TOTAL:PASS | |
| # total 111M | |
| # 0 drwxr-xr-x 2 user user 240 2022-04-05 13:38:03 . | |
| # 0 drwxrwxrwt 12 user user 300 2022-04-05 13:56:24 .. | |
| # 216K -rw-r--r-- 1 user user 213K 2022-04-05 13:37:55 fatrace.log | |
| # 28K -rw-r--r-- 1 user user 28K 2022-04-05 13:37:55 process_A.yt_dlp.log | |
| # 16K -rw-r--r-- 1 user user 13K 2022-04-05 13:37:55 process_B.yt_dlp.log | |
| # 8.0K -rw-r--r-- 1 user user 5.3K 2022-04-05 13:38:12 terminal_log.txt | |
| # 3.7M -rwxr-xr-x 1 user user 3.7M 2018-11-15 16:10:08 youtube__Gwo3pEH7hUE.f139.m4a | |
| # 4.0M -rwxr-xr-x 1 user user 4.0M 2018-11-15 16:13:45 youtube__Gwo3pEH7hUE.f160.mp4 | |
| # 7.9M -rw-r--r-- 1 user user 7.9M 2022-04-05 13:38:03 youtube__Gwo3pEH7hUE.manual.mp4 | |
| # 7.9M -rw-r--r-- 1 user user 7.9M 2018-11-15 16:10:08 youtube__Gwo3pEH7hUE.mp4 | |
| # 28K -rwxr-xr-x 1 user user 26K 2022-04-05 13:36:51 yt-dlp-advisory-locking-test.sh | |
| # 87M -rw-r--r-- 1 user user 87M 2022-04-05 13:38:03 yt-dlp.strace.log | |
| # | |
| # $ ls _yt_dlp_test_1649191734___HEAD:554800c85___VIDEO:PASS__AUDIO:PASS__AUTOMERGE:fail__MANUALMERGE:PASS__TOTAL:fail | |
| # total 88M | |
| # 0 drwxr-xr-x 2 user user 220 2022-04-05 13:50:03 . | |
| # 0 drwxrwxrwt 12 user user 300 2022-04-05 13:56:24 .. | |
| # 172K -rw-r--r-- 1 user user 171K 2022-04-05 13:49:56 fatrace.log | |
| # 28K -rw-r--r-- 1 user user 28K 2022-04-05 13:49:56 process_A.yt_dlp.log | |
| # 16K -rw-r--r-- 1 user user 13K 2022-04-05 13:49:56 process_B.yt_dlp.log | |
| # 8.0K -rw-r--r-- 1 user user 5.1K 2022-04-05 13:50:07 terminal_log.txt | |
| # 3.7M -rwxr-xr-x 1 user user 3.7M 2018-11-15 16:10:08 youtube__Gwo3pEH7hUE.f139.m4a | |
| # 4.0M -rwxr-xr-x 1 user user 4.0M 2018-11-15 16:13:45 youtube__Gwo3pEH7hUE.f160.mp4 | |
| # 7.9M -rw-r--r-- 1 user user 7.9M 2022-04-05 13:50:03 youtube__Gwo3pEH7hUE.manual.mp4 | |
| # 28K -rwxr-xr-x 1 user user 26K 2022-04-05 13:48:54 yt-dlp-advisory-locking-test.sh | |
| # 72M -rw-r--r-- 1 user user 72M 2022-04-05 13:50:03 yt-dlp.strace.log | |
| # other race condition cases (incomplete list) | |
| # | |
| # 1. "nobody merged" (the unrelated "AUTOMAEGE:fail" issue above) | |
| # B: locks and starts writing youtube__Gwo3pEH7hUE.f160.mp4 | |
| # A: BlockingIOError trying to lock youtube__Gwo3pEH7hUE.f160.mp4 | |
| # A: locks and starts downloading youtube__Gwo3pEH7hUE.f139.m4a | |
| # B: BlockingIOError trying to lock youtube__Gwo3pEH7hUE.f139.m4a | |
| # B: does not attempt to merge since B did not download the audio track | |
| # A: does not attempt to merge since A did not download the video track | |
| # 1a. youtube__Gwo3pEH7hUE.f160.mp4 and youtube__Gwo3pEH7hUE.f139.m4a exist | |
| # | |
| # 2. "the process that downloaded the video tried to merge" | |
| # (either after BlockingIOError on the audio, or not) | |
| # | |
| # 3. "not holding locks until done with merge" | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment