From 3012c17b17334f298920f24b95f2be5f41656761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20L=C3=B8vbr=C3=B8tte=20Olsen?= Date: Fri, 27 Mar 2020 01:50:01 +0100 Subject: [PATCH] MVP --- Cargo.lock | 12 +++++++++ Cargo.toml | 1 + client.sh | 65 +++++++++++++++++++++++++++++++++++++++++++++++ create_jobs.sh | 41 ++++++++++++++++++++++++++++++ src/main.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++++++- src/workunit.rs | 8 ++++-- test.sh | 44 ++++++++++++++++++++++++++++++++ 7 files changed, 235 insertions(+), 3 deletions(-) create mode 100755 client.sh create mode 100755 create_jobs.sh create mode 100755 test.sh diff --git a/Cargo.lock b/Cargo.lock index 1a075b1..eee2342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,7 @@ dependencies = [ "rocket_contrib", "serde", "serde_json", + "serde_repr", "uuid", ] @@ -749,6 +750,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd02c7587ec314570041b2754829f84d873ced14a96d1fd1823531e11db40573" +dependencies = [ + "proc-macro2 1.0.9", + "quote 1.0.3", + "syn 1.0.17", +] + [[package]] name = "slab" version = "0.4.2" diff --git a/Cargo.toml b/Cargo.toml index 76e3988..87108f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ rocket = "0.4.4" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" +serde_repr = "0.1" uuid = { version = "0.7", features = ["serde", "v4"] } diff --git a/client.sh b/client.sh new file mode 100755 index 0000000..b2bfb54 --- /dev/null +++ b/client.sh @@ -0,0 +1,65 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl jq libaom ffmpeg-full + +set -euo pipefail +IFS=$'\n\t' + +base_url="http://localhost:8000" +version="0.1.0" + +while true; do + sleep 1 + upsteam_version=`curl -s "$base_url"/version` + if [[ $version != $upsteam_version ]]; then + break + fi + + job=`curl -s "$base_url"/request_job | jq` + if [[ $job = "null" ]]; then + echo "No Jobs Available ¯\_(ツ)_/¯" + continue + fi + + echo "Got new job!" + echo "$job" | jq + + job_id=`echo "$job" | jq -r .id` + + echo "Reserving Job" + curl -s "$base_url"/edit_status/"$job_id"/reserved + echo "" + + source=`echo $job | jq -r .description.file_url` + sourceext=${source##*.} + echo "Downloading source file: $source" + + source=`echo $job | jq -r .description.file_url` + + name=`echo $job | jq -r .description.file_name` + input="$name.$job_id.$sourceext" + + curl "$source" -o "$input" + echo "" + + echo "Starting Encode" + + target_bitrate=`echo $job | jq -r .description.options.mode.VBR` + width=`echo $job | jq -r .description.options.resolution.width` + height=`echo $job | jq -r .description.options.resolution.height` + color_depth=`echo $job | jq -r .description.options.color_depth` + kf_min_dist=`echo $job | jq -r .description.options.kf_min_dist` + kf_max_dist=`echo $job | jq -r .description.options.kf_max_dist` + + speed=`echo $job | jq -r .description.options.speed` + + ffmpeg -i "$input" -vf scale=$width:$height -f yuv4mpegpipe - | aomenc - --lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 \ + --target-bitrate=$target_bitrate --width="$width" --height="$height" --bit-depth=$color_depth --kf-min-dist=$kf_min_dist --kf-max-dist=$kf_min_dist \ + --cpu-used=$speed \ + --pass=1 --passes=2 --fpf="$input.$target_bitrate.$width.$height.$color_depth.fpf" --webm -o "$input.$target_bitrate.$width.$height.$color_depth.webm" + + ffmpeg -i "$input" -vf scale=$width:$height -f yuv4mpegpipe - | aomenc - --lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 \ + --target-bitrate=$target_bitrate --width="$width" --height="$height" --bit-depth=$color_depth --kf-min-dist=$kf_min_dist --kf-max-dist=$kf_min_dist \ + --cpu-used=$speed \ + --pass=2 --passes=2 --fpf="$input.$target_bitrate.$width.$height.$color_depth.fpf" --webm -o "$input.$target_bitrate.$width.$height.$color_depth.webm" + +done \ No newline at end of file diff --git a/create_jobs.sh b/create_jobs.sh new file mode 100755 index 0000000..0dcf8f9 --- /dev/null +++ b/create_jobs.sh @@ -0,0 +1,41 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p curl bash ffmpeg-full + +set -euo pipefail +IFS=$'\n\t' + +base_url="http://localhost:8000" + +upload() { for f; do echo $(curl -#Sf -F "files[]=@$f" https://pomf.dodsorf.as/upload\?output\=text); done } + +files=`find $1 -name "*.mkv" -type f` + +for file in $files; do + file_url=`upload $file` + file_name=`basename $file .mkv` + length=`ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of default=nokey=1:noprint_wrappers=1 $file` + + curl "$base_url"/add_job -X POST -H "Content-Type: application/json" -d \ + ' + { + "file_url": "'$file_url'", + "file_name": "'$file_name'", + "priority": 0, + "length": '$length', + "options": { + "mode": { "VBR": 33 }, + "color_depth": 10, + "enable_fwd_keyframe": true, + "two_pass": true, + "speed": 0, + "resolution": { + "width": 960, + "height": 540 + }, + "kf_min_dist": 9999, + "kf_max_dist": 9999 + } + } + ' +done + diff --git a/src/main.rs b/src/main.rs index 343a8c4..1d2a29c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,72 @@ struct SharedState { #[get("/")] fn index() -> &'static str { - "Welcome to the AV1 Encoder Master Server" +r#"#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl jq libaom ffmpeg-full + +set -euo pipefail +IFS=$'\n\t' + +base_url="http://localhost:8000" +version="0.1.0" + +while true; do + sleep 1 + upsteam_version=`curl -s "$base_url"/version` + if [[ $version != $upsteam_version ]]; then + break + fi + + job=`curl -s "$base_url"/request_job | jq` + if [[ $job = "null" ]]; then + echo "No Jobs Available ¯\_(ツ)_/¯" + continue + fi + + echo "Got new job!" + echo "$job" | jq + + job_id=`echo "$job" | jq -r .id` + + echo "Reserving Job" + curl -s "$base_url"/edit_status/"$job_id"/reserved + echo "" + + source=`echo $job | jq -r .description.file_url` + sourceext=${source##*.} + echo "Downloading source file: $source" + + source=`echo $job | jq -r .description.file_url` + + name=`echo $job | jq -r .description.file_name` + input="$name.$job_id.$sourceext" + + curl "$source" -o "$input" + echo "" + + echo "Starting Encode" + + target_bitrate=`echo $job | jq -r .description.options.mode.VBR` + width=`echo $job | jq -r .description.options.resolution.width` + height=`echo $job | jq -r .description.options.resolution.height` + color_depth=`echo $job | jq -r .description.options.color_depth` + kf_min_dist=`echo $job | jq -r .description.options.kf_min_dist` + kf_max_dist=`echo $job | jq -r .description.options.kf_max_dist` + + speed=`echo $job | jq -r .description.options.speed` + + ffmpeg -i "$input" -vf scale=$width:$height -f yuv4mpegpipe - | aomenc - --lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 \ + --target-bitrate=$target_bitrate --width="$width" --height="$height" --bit-depth=$color_depth --kf-min-dist=$kf_min_dist --kf-max-dist=$kf_min_dist \ + --cpu-used=$speed \ + --pass=1 --passes=2 --fpf="$input.$target_bitrate.$width.$height.$color_depth.fpf" --webm -o "$input.$target_bitrate.$width.$height.$color_depth.webm" + + ffmpeg -i "$input" -vf scale=$width:$height -f yuv4mpegpipe - | aomenc - --lag-in-frames=25 --tile-columns=0 --tile-rows=0 --enable-fwd-kf=1 \ + --target-bitrate=$target_bitrate --width="$width" --height="$height" --bit-depth=$color_depth --kf-min-dist=$kf_min_dist --kf-max-dist=$kf_min_dist \ + --cpu-used=$speed \ + --pass=2 --passes=2 --fpf="$input.$target_bitrate.$width.$height.$color_depth.fpf" --webm -o "$input.$target_bitrate.$width.$height.$color_depth.webm" + +done +"# } #[get("/version")] diff --git a/src/workunit.rs b/src/workunit.rs index d7a1da7..51c8a18 100644 --- a/src/workunit.rs +++ b/src/workunit.rs @@ -1,4 +1,5 @@ use serde::{Serialize, Deserialize}; +use serde_repr::*; use uuid::Uuid; #[derive(Default, Debug, Serialize, Deserialize, Clone)] @@ -20,14 +21,16 @@ impl WUnit { #[derive(Default, Debug, Serialize, Deserialize, Clone)] pub struct WDesc { pub file_url: String, + pub file_name: String, pub priority: u16, pub length: u32, pub options: EOptions, } impl WDesc { - pub fn new(file_url: &str, priority: Option, length: u32, options: Option) -> Self { + pub fn new(file_url: &str, file_name: &str, priority: Option, length: u32, options: Option) -> Self { WDesc { file_url: file_url.to_string(), + file_name: file_name.to_string(), priority: priority.unwrap_or(0), length: length, options: options.unwrap_or(EOptions::default()), @@ -82,7 +85,8 @@ pub struct Resolution { pub height: u16 } -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[derive(Debug, Serialize_repr, Deserialize_repr, PartialEq, Clone)] +#[repr(u8)] pub enum EColorDepth { Eight = 8, Ten = 10, diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..3677835 --- /dev/null +++ b/test.sh @@ -0,0 +1,44 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl jq + +base_url="http://localhost:8000" + +curl "$base_url"/version +curl "$base_url" + +curl "$base_url"/get_jobs | jq + +curl "$base_url"/add_job -X POST -H "Content-Type: application/json" -d \ +' + { + "file_url": "https://pomf.dodsorf.as/f/38ez7v.mkv", + "file_name": "014", + "priority": 0, + "length": 15, + "options": { + "mode": { "VBR": 33 }, + "color_depth": 10, + "enable_fwd_keyframe": true, + "two_pass": true, + "speed": 0, + "resolution": { + "width": 960, + "height": 540 + }, + "kf_min_dist": 9999, + "kf_max_dist": 9999 + } + } +' + +curl "$base_url"/request_job | jq +job=`curl "$base_url"/request_job | jq -r .id` + +curl "$base_url"/get_job/"$job" | jq + +curl "$base_url"/edit_status/"$job"/reserved +curl "$base_url"/edit_status/"$job"/queued + +curl "$base_url"/get_job/"$job" | jq + +curl "$base_url"/request_job | jq \ No newline at end of file