From 6e893006759383aafd68ef4479a234c90c8afc12 Mon Sep 17 00:00:00 2001 From: Daniel Olsen Date: Wed, 13 May 2020 15:23:34 +0200 Subject: [PATCH] 0.11.0 WIP Former-commit-id: 94fcad7d7cf1f7733a2f773572c1b0990a31232d --- src/main.rs | 12 ++- src/static/client.sh | 171 +++++++++++++++++++++++++------------------ src/workunit.rs | 45 ++++++++++-- test.sh | 14 ++-- 4 files changed, 156 insertions(+), 86 deletions(-) diff --git a/src/main.rs b/src/main.rs index 110ddd0..1efd600 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ mod workunit; use workunit::WUnit; use workunit::EStatus; -const VERSION: &str = "0.10.0"; +const VERSION: &str = "0.11.0"; #[derive(Default, Debug)] struct SharedState { @@ -30,7 +30,7 @@ struct SharedState { fn index() -> String { format!("Wecome to the AV1Master Server version {version}\n This currently requires a distro with CAP_SYS_USER_NS enabled and correct permissions -curl -L {baseurl}/av1client > av1client && chmod +x ./av1client && ./av1client", baseurl="https://av1.dodsorf.as", version=VERSION) +curl -L {baseurl}/av1client > av1client && chmod +x ./av1client && ./av1client {baseurl}", baseurl="https://av1.dodsorf.as", version=VERSION) } #[get("/version")] @@ -134,5 +134,13 @@ fn main() { .manage(SharedState::default()) .mount("/", StaticFiles::from("src/static")) // switch to templates or something cause this is dumb .mount("/", routes![index, version, get_jobs, get_job, request_job, edit_status, add_job, upload]) + .mount("/", routes![test_job]) .launch(); } + + +#[get("/test_job/")] +fn test_job(jobset: String, shared: State) { + let id = uuid::Uuid::new_v4(); + shared.jobs.lock().unwrap().insert(id, WUnit::new(id, jobset, workunit::WDesc::new("https://pomf.dodsorf.as/f/vz9dtl.mkv", "014", None, 90, (540, 960), None))); +} \ No newline at end of file diff --git a/src/static/client.sh b/src/static/client.sh index 01ccb37..be9a425 100644 --- a/src/static/client.sh +++ b/src/static/client.sh @@ -3,8 +3,8 @@ set -euo pipefail IFS=$'\n\t' -base_url="https://av1.dodsorf.as" -version="0.10.0" +base_url="$1" +version="0.11.0" while true; do sleep 30 @@ -71,96 +71,121 @@ while true; do echo "Starting Encode" + etype=`echo $job | jq -r '.description.options | keys | .[]'` + + if [$etype != "FFMPEG"] || [$etype != "AOMENC" ]; then + echo "That's not a valid encoder!! Are you being attacked?" + fi + height=`echo $job | jq -r .description.resolution[0]` width=`echo $job | jq -r .description.resolution[1]` echo $job | jq - aomenco=`echo $job | jq -r .description.options.aomenc` - aomenco=${aomenco//[^a-zA-Z0-9_\- =]/} - ffmpego=`echo $job | jq -r .description.options.ffmpeg` - ffmpego=${ffmpego//[^a-zA-Z0-9_\- =:]/} + options=`echo $job | jq .description.options.$etype` - pix_fmt=`echo $job | jq -r .description.options.pix_fmt` - if [[ $pix_fmt = "YV12" ]]; then - ffpix="yuv12p" - aompix="--yv12" - elif [[ $pix_fmt = "I420" ]]; then - ffpix="yuv420p" - aompix="--i420" - elif [[ $pix_fmt = "I422" ]]; then - ffpix="yuv422p" - aompix="--i422" - elif [[ $pix_fmt = "I444" ]]; then - ffpix="yuv444p" - aompix="--i444" - fi + case $etype in + AOMENC) + aomenco=`echo $options | jq -r .aomenc` + aomenco=${aomenco//[^a-zA-Z0-9_\- =]/} + ffmpego=`echo $options | jq -r .ffmpeg` + ffmpego=${ffmpego//[^a-zA-Z0-9_\- =:]/} + + pix_fmt=`echo $job | jq -r .pix_fmt` + if [[ $pix_fmt = "YV12" ]]; then + ffpix="yuv12p" + aompix="--yv12" + elif [[ $pix_fmt = "I420" ]]; then + ffpix="yuv420p" + aompix="--i420" + elif [[ $pix_fmt = "I422" ]]; then + ffpix="yuv422p" + aompix="--i422" + elif [[ $pix_fmt = "I444" ]]; then + ffpix="yuv444p" + aompix="--i444" + fi - fps=`echo $job | jq -r .description.options.fps` - if [[ $fps = "null" ]]; then - fffps="" - aomfps="" - else - fpsrate=`echo $fps | jq -r '.[0]'` - fpsscale=`echo $fps | jq -r '.[1]'` - fpsv="$fpsrate/$fpsscale" - fffps="fps=fps=$fpsv -r $fpsv" - aomfps="--fps=$fpsv" - fi + fps=`echo $options | jq -r .fps` + if [[ $fps = "null" ]]; then + fffps="" + aomfps="" + else + fpsrate=`echo $fps | jq -r '.[0]'` + fpsscale=`echo $fps | jq -r '.[1]'` + fpsv="$fpsrate/$fpsscale" + fffps="fps=fps=$fpsv -r $fpsv" + aomfps="--fps=$fpsv" + fi + two_pass=`echo $options | jq -r .two_pass` - two_pass=`echo $job | jq -r .description.options.two_pass` + if [[ $two_pass = true ]]; then + set +e + eval 'ffmpeg -nostats -hide_banner -loglevel warning \ + -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ + --pass=1 --passes=2 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' - if [[ $two_pass = true ]]; then - set +e - eval 'ffmpeg -nostats -hide_banner -loglevel warning \ - -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ - --pass=1 --passes=2 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' + retval=$? + if [ $retval -ne 0 ]; then + echo "Error running encode pass 1" + curl -s -L "$base_url"/edit_status/"$job_id"/error || true + echo "" + continue + fi - retval=$? - if [ $retval -ne 0 ]; then - echo "Error running encode pass 1" - curl -s -L "$base_url"/edit_status/"$job_id"/error || true - echo "" - continue - fi + eval 'ffmpeg -nostats -hide_banner -loglevel warning \ + -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ + --pass=2 --passes=2 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' - eval 'ffmpeg -nostats -hide_banner -loglevel warning \ - -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ - --pass=2 --passes=2 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' + retval=$? + if [ $retval -ne 0 ]; then + echo "Error running encode pass 2" + curl -s -L "$base_url"/edit_status/"$job_id"/error || true + echo "" + continue + fi + set -e - retval=$? - if [ $retval -ne 0 ]; then - echo "Error running encode pass 2" - curl -s -L "$base_url"/edit_status/"$job_id"/error || true - echo "" - continue - fi - set -e + echo "Deleting Source and Temporary files" + rm "$input" "$input".fpf - echo "Deleting Source and Temporary files" - rm "$input" "$input".fpf + else + set +e + eval 'ffmpeg -nostats -hide_banner -loglevel warning \ + -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ + --passes=1 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' - else - set +e - eval 'ffmpeg -nostats -hide_banner -loglevel warning \ - -i "'$input'" '$ffmpego' -vf scale='$width':'$height','$fffps' -pix_fmt '$ffpix' -f yuv4mpegpipe - | aomenc - '$aomfps' '$aompix' '$aomenco' \ - --passes=1 --fpf="'$input'.fpf" --ivf -o "'$input'.out.ivf"' + retval=$? + if [ $retval -ne 0 ]; then + echo "Error running encode" + curl -s -L "$base_url"/edit_status/"$job_id"/error || true + echo "" + continue + fi + set -e - retval=$? - if [ $retval -ne 0 ]; then - echo "Error running encode" - curl -s -L "$base_url"/edit_status/"$job_id"/error || true - echo "" - continue - fi - set -e + echo "Deleting Source" + rm "$input" + fi + ;; + FFMPEG) + echo "Starting FFMPEG encode" + if [[ $two_pass = true ]]; then + echo "Running in two-pass mode" + + options=`echo $job | jq -r .options.$etype` - echo "Deleting Source" - rm "$input" - fi + pix_fmt=`echo $options | jq -r .pix_fmt` + tiles=`echo $options | jq -r .tiles` + lag_in_frames=`echo $options | jq -r .lag_in_frames` + + set +e + ffmpeg -i $input -c:v libaom-av1 -strict experimental -pass 1 -an -vf scale=$width:$height -pix_fmt $pix_fmt -tiles $tiles -lag-in-frames $lag_in_frames $flag_g -f ivf /dev/null + fi + ;; set +e curl -s -L "$base_url"/edit_status/"$job_id"/completed diff --git a/src/workunit.rs b/src/workunit.rs index 7d2f713..3ab0c97 100644 --- a/src/workunit.rs +++ b/src/workunit.rs @@ -49,27 +49,62 @@ impl WDesc { } } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum EOptions { + AOMENC(AomencO), + FFMPEG(FffmpegO) +} +impl Default for EOptions { + fn default() -> Self { + let default = AomencO::default(); + EOptions::AOMENC(default) + } +} #[derive(Debug, Serialize, Deserialize, Clone)] -pub struct EOptions { +pub struct AomencO { pub ffmpeg: String, pub aomenc: String, pub two_pass: bool, pub pix_fmt: EPixFmt, pub fps: (u16, u16) } -impl Default for EOptions { +impl Default for AomencO { fn default() -> Self { - EOptions{ + AomencO{ ffmpeg: String::default(), - aomenc: "--lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 --bit-depth=10 --cpu-used=3 --cq-level=30 --end-usage=q".to_string(), - two_pass: false, + aomenc: "--lag-in-frames=35 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 --bit-depth=10 --cpu-used=4 --cq-level=30 --end-usage=q".to_string(), + two_pass: true, pix_fmt: EPixFmt::I422, fps: (25, 1) } } } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct FffmpegO { + pub two_pass: bool, + pub crf: u8, + pub b_v: String, + pub lag_in_frames: u8, + pub pix_fmt: String, + pub tiles: String, + pub speed: u8 +} +impl Default for FffmpegO { + fn default() -> Self { + FffmpegO { + two_pass: true, + crf: 30, + b_v: "0".to_string(), + lag_in_frames: 35, + pix_fmt: "yuv420p10le" + tiles: "1x1".to_string(), + speed: 4 + } + } +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub enum EStatus { Queued, diff --git a/test.sh b/test.sh index ae48210..19cd124 100755 --- a/test.sh +++ b/test.sh @@ -8,7 +8,7 @@ curl "$base_url" curl "$base_url"/get_jobs | jq -curl "$base_url"/add_job -X POST -H "Content-Type: application/json" -d \ +curl "$base_url"/add_job/b -X POST -H "Content-Type: application/json" -d \ ' { "file_url": "https://pomf.dodsorf.as/f/vz9dtl.mkv", @@ -17,11 +17,13 @@ curl "$base_url"/add_job -X POST -H "Content-Type: application/json" -d \ "length": 90, "resolution": [540, 960], "options": { - "aomenc": "--lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 --bit-depth=10 --cpu-used=0 --end-usage=vbr --target-bitrate=60 --kf-min-dist=9999 --kf-max-dist=9999", - "ffmpeg": "", - "two_pass": true, - "pix_fmt": "YV12", - "fps": [24000, 1001] + "FFMPEG": { + "two_pass": true, + "crf": 45, + "b_v": "0", + "tiles": "1x1", + "speed": 4 + } } } '