diff --git a/api/extractvideopreviews.php b/api/extractvideopreviews.php index 38f77ac..1cef1ef 100755 --- a/api/extractvideopreviews.php +++ b/api/extractvideopreviews.php @@ -2,266 +2,16 @@ require_once './src/Database.php'; require_once './src/TMDBMovie.php'; require_once './src/SSettings.php'; +require_once './src/VideoParser.php'; // allow UTF8 characters setlocale(LC_ALL, 'en_US.UTF-8'); -writeLog("starting extraction!\n"); +$vp = new VideoParser(); +$vp->writeLog("starting extraction!!\n"); -$ffmpeg = 'ffmpeg'; //or: /usr/bin/ffmpeg , or /usr/local/bin/ffmpeg - depends on your installation (type which ffmpeg into a console to find the install path) -$tmdb = new TMDBMovie(); -// initial load of all available movie genres -$tmdbgenres = $tmdb->getAllGenres(); - -$conn = Database::getInstance()->getConnection(); -$settings = new SSettings(); +$sett = new SSettings(); // load video path from settings -$scandir = "../" . $settings->getVideoPath(); -$arr = scandir($scandir); - -$all = 0; -$added = 0; -$deleted = 0; -$failed = 0; - -foreach ($arr as $elem) { - $ext = pathinfo($elem, PATHINFO_EXTENSION); - if ($ext == "mp4") { - $moviename = substr($elem, 0, -4); - - $query = "SELECT * FROM videos WHERE movie_name = '" . mysqli_real_escape_string($conn, $moviename) . "'"; - $result = $conn->query($query); - - // insert if not available in db - if (!mysqli_fetch_assoc($result)) { - $genres = -1; - $poster = -1; // initially disable poster supp - // extract poster from video - $pic = shell_exec("ffmpeg -hide_banner -loglevel panic -ss 00:04:00 -i \"../videos/prn/$elem\" -vframes 1 -q:v 2 -f singlejpeg pipe:1 2>/dev/null"); - // check if tmdb grabbing is enabled - if ($settings->isTMDBGrabbingEnabled()) { - // search in tmdb api - if (!is_null($dta = $tmdb->searchMovie($moviename))) { - $poster = file_get_contents($tmdb->picturebase . $dta->poster_path); - - // error handling for download error - if (!$poster) { - echo "Failed to load Picture from TMDB! \n"; - } - // store genre ids for parsing later - $genres = $dta->genre_ids; - } else { - // nothing found with tmdb - echo "nothing found with TMDB!\n"; - writeLog("nothing found with TMDB!\n"); - } - } - - // convert video to base64 - $image = 'data:image/jpeg;base64,' . base64_encode($pic); - - // extract other video attributes - $video_attributes = _get_video_attributes($elem); - $duration = 0; - $size = 0; - $width = 0; - - if ($video_attributes) { - $duration = $video_attributes->media->track[0]->Duration; // in seconds - $size = $video_attributes->media->track[0]->FileSize; // in Bytes - $width = $video_attributes->media->track[1]->Width; // width - } - - - if (!$poster) { - $poster_base64 = 'data:image/jpeg;base64,' . base64_encode($poster); - - $query = "INSERT INTO videos(movie_name,movie_url,poster,thumbnail,quality,length) - VALUES ('" . mysqli_real_escape_string($conn, $moviename) . "', - '" . mysqli_real_escape_string($conn, 'videos/prn/' . $elem) . "', - '$poster_base64', - '$image', - '$width', - '$duration')"; - } else { - $query = "INSERT INTO videos(movie_name,movie_url,thumbnail,quality,length) - VALUES ('" . mysqli_real_escape_string($conn, $moviename) . "', - '" . mysqli_real_escape_string($conn, 'videos/prn/' . $elem) . "', - '$image', - '$width', - '$duration')"; - } - - - if ($conn->query($query) === TRUE) { - echo('successfully added ' . $elem . " to video gravity\n"); - writeLog('successfully added ' . $elem . " to video gravity\n"); - - // add this entry to the default tags - $last_id = $conn->insert_id; - - // full hd - if ($width >= 1900) { - $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,2)"; - if ($conn->query($query) !== TRUE) { - echo "failed to add default tag here.\n"; - writeLog("failed to add default tag here.\n"); - } - } - - // HD - if ($width >= 1250 && $width < 1900) { - $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,4)"; - if ($conn->query($query) !== TRUE) { - echo "failed to add default tag here.\n"; - writeLog("failed to add default tag here.\n"); - } - } - - // SD - if ($width < 1250 && $width > 0) { - $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,3)"; - if ($conn->query($query) !== TRUE) { - echo "failed to add default tag here.\n"; - writeLog("failed to add default tag here.\n"); - } - } - - // handle tmdb genres here! - if ($genres != -1) { - // transform genre ids in valid names - foreach ($genres as $genreid) { - // check if genre is already a tag in db if not insert it - $tagname = array_column($tmdbgenres, 'name', 'id')[$genreid]; - $tagid = tagExists($tagname); - - $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,$tagid)"; - if ($conn->query($query) !== TRUE) { - echo "failed to add $genreid tag here.\n"; - writeLog("failed to add $genreid tag here.\n"); - } - } - } - - - $added++; - $all++; - } else { - echo('errored item: ' . $elem . "\n"); - writeLog('errored item: ' . $elem . "\n"); - echo('{"data":"' . $conn->error . '"}\n'); - writeLog('{"data":"' . $conn->error . '"}\n'); - $failed++; - } - } else { - $all++; - } - } else { - echo($elem . " does not contain a .mp4 extension! - skipping \n"); - writeLog($elem . " does not contain a .mp4 extension! - skipping \n"); - } -} - -// auto cleanup db entries -$query = "SELECT COUNT(*) as count FROM videos"; -$result = $conn->query($query); -$r = mysqli_fetch_assoc($result); - -if ($all < $r['count']) { - echo "should be in gravity: " . $all . "\n"; - writeLog("should be in gravity: " . $all . "\n"); - echo "really in gravity: " . $r['count'] . "\n"; - writeLog("really in gravity: " . $r['count'] . "\n"); - echo "cleaning up gravity\n"; - writeLog("cleaning up gravity\n"); - - $query = "SELECT movie_id,movie_url FROM videos"; - $result = $conn->query($query); - - while ($r = mysqli_fetch_assoc($result)) { - if (!file_exists("../" . $r['movie_url'])) { - $query = "SET foreign_key_checks = 0; DELETE FROM videos WHERE movie_id='" . $r['movie_id'] . "'"; - if ($conn->query($query) === TRUE) { - echo('successfully deleted ' . $r['movie_url'] . " from video gravity\n"); - writeLog('successfully deleted ' . $r['movie_url'] . " from video gravity\n"); - $deleted++; - } else { - echo "failed to delete " . $r['movie_url'] . " from gravity: " . $conn->error . "\n"; - writeLog("failed to delete " . $r['movie_url'] . " from gravity: " . $conn->error . "\n"); - } - } - } -} - -// calculate size of databse here -$size = -1; -$query = "SELECT table_schema AS \"Database\", - ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS \"Size\" - FROM information_schema.TABLES - WHERE TABLE_SCHEMA='" . Database::getInstance()->getDatabaseName() . "' - GROUP BY table_schema;"; -$result = $conn->query($query); -if ($result->num_rows == 1) { - $row = $result->fetch_assoc(); - $size = $row["Size"]; -} - -echo "Total gravity: " . $all . "\n"; -writeLog("Total gravity: " . $all . "\n"); -echo "Size of Databse is: " . $size . "MB\n"; -writeLog("Size of Databse is: " . $size . "MB\n"); -echo "added in this run: " . $added . "\n"; -writeLog("added in this run: " . $added . "\n"); -echo "deleted in this run: " . $deleted . "\n"; -writeLog("deleted in this run: " . $deleted . "\n"); -echo "errored in this run: " . $failed . "\n"; -writeLog("errored in this run: " . $failed . "\n"); - -writeLog("-42"); // terminating characters to stop webui requesting infos - -/** - * get all videoinfos of a video file - * - * @param $video string name including extension - * @return object all infos as object - */ -function _get_video_attributes($video) { - $command = "mediainfo \"../videos/prn/$video\" --Output=JSON"; - $output = shell_exec($command); - return json_decode($output); -} - -/** - * write a line to the output log file - * - * @param string $message message to write - */ -function writeLog(string $message) { - file_put_contents("/tmp/output.log", $message, FILE_APPEND); - flush(); -} - -/** - * ckecks if tag exists -- if not creates it - * @param string $tagname the name of the tag - * @return integer the id of the inserted tag - */ -function tagExists(string $tagname) { - global $conn; - - $query = "SELECT * FROM tags WHERE tag_name='$tagname'"; - - $result = $conn->query($query); - if ($result->num_rows == 0) { - // tag does not exist --> create it - $query = "INSERT INTO tags (tag_name) VALUES ('$tagname')"; - if ($conn->query($query) !== TRUE) { - echo "failed to create $tagname tag in database\n"; - writeLog("failed to create $tagname tag in database\n"); - } - return $conn->insert_id; - } else { - return $result->fetch_assoc()['tag_id']; - } -} +$scandir = "../" . $sett->getVideoPath(); +$vp->extractVideos($scandir); diff --git a/api/src/SSettings.php b/api/src/SSettings.php index 8d32ca2..97b3c41 100644 --- a/api/src/SSettings.php +++ b/api/src/SSettings.php @@ -32,7 +32,7 @@ class SSettings { * @return bool isenabled? */ public function isTMDBGrabbingEnabled(): bool { - $query = "SELECT TMDB_grabbing from settings"; + $query = "SELECT TMDB_grabbing from settings WHERE 1"; $result = $this->database->getConnection()->query($query); if (!$result) { diff --git a/api/src/TMDBMovie.php b/api/src/TMDBMovie.php index 5d481d3..1c8233b 100644 --- a/api/src/TMDBMovie.php +++ b/api/src/TMDBMovie.php @@ -15,12 +15,25 @@ class TMDBMovie { * @param string $moviename moviename * @return object movie object or null if not found */ - public function searchMovie(string $moviename) { + public function searchMovie(string $moviename, string $year = null) { $reply = json_decode(file_get_contents($this->baseurl . "search/movie?api_key=" . $this->apikey . "&query=" . urlencode($moviename))); if ($reply->total_results == 0) { // no results found - // todo maybe parse first pictures somehow return null; + } elseif ($year != null) { + // if year is defined check year + $regex = '/[0-9]{4}?/'; // matches year of string + + for ($i = 0; $i < count($reply->results); $i++) { + $releasedate = $reply->results[$i]->release_date; + + preg_match($regex, $releasedate, $matches); + if (count($matches) > 0) { + $curryear = $matches[0]; + if ($curryear == $year) + return $reply->results[$i]; + } + } } else { return $reply->results[0]; } diff --git a/api/src/VideoParser.php b/api/src/VideoParser.php new file mode 100644 index 0000000..982371a --- /dev/null +++ b/api/src/VideoParser.php @@ -0,0 +1,334 @@ +tmdb = new TMDBMovie(); + $this->tmdbgenres = $this->tmdb->getAllGenres(); + $this->conn = Database::getInstance()->getConnection(); + + $this->settings = new SSettings(); + $this->TMDBenabled = $this->settings->isTMDBGrabbingEnabled(); + $this->videopath = $this->settings->getVideoPath(); + } + + /** + * searches a folder for mp4 videos and adds them to video gravity + * @param $foldername string the folder where to search (relative to the webserver root) + */ + public function extractVideos(string $foldername) { + echo("TMDB grabbing is " . ($this->TMDBenabled ? "" : "not") . " enabled \n"); + $arr = scandir($foldername); + + foreach ($arr as $elem) { + if($elem == '.' || $elem == '..') continue; + + $ext = pathinfo($elem, PATHINFO_EXTENSION); + if ($ext == "mp4") { + $this->processVideo($elem); + } else { + echo($elem . " does not contain a valid .mp4 extension! - skipping \n"); + $this->writeLog($elem . " does not contain a valid .mp4 extension! - skipping \n"); + } + } + + // cleanup gravity + $this->cleanUpGravity(); + + // calculate size of databse here + $size = -1; + $query = "SELECT table_schema AS \"Database\", + ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS \"Size\" + FROM information_schema.TABLES + WHERE TABLE_SCHEMA='" . Database::getInstance()->getDatabaseName() . "' + GROUP BY table_schema;"; + $result = $this->conn->query($query); + if ($result->num_rows == 1) { + $row = $result->fetch_assoc(); + $size = $row["Size"]; + } + + echo "Total gravity: " . $this->all . "\n"; + $this->writeLog("Total gravity: " . $this->all . "\n"); + echo "Size of Databse is: " . $size . "MB\n"; + $this->writeLog("Size of Databse is: " . $size . "MB\n"); + echo "added in this run: " . $this->added . "\n"; + $this->writeLog("added in this run: " . $this->added . "\n"); + echo "deleted in this run: " . $this->deleted . "\n"; + $this->writeLog("deleted in this run: " . $this->deleted . "\n"); + echo "errored in this run: " . $this->failed . "\n"; + $this->writeLog("errored in this run: " . $this->failed . "\n"); + + $this->writeLog("-42"); // terminating characters to stop webui requesting infos + } + + /** + * processes one mp4 video, extracts tags and adds it to the database + * @param $filename string filename of the video to process + */ + private function processVideo(string $filename) { + $moviename = substr($filename, 0, -4); + + $regex = '/\([0-9]{4}?\)/'; //match year pattern + preg_match($regex, $moviename, $matches); + preg_replace($regex, '', $moviename); + $year = null; + if (count($matches) > 0) { + $year = substr($matches[count($matches) - 1], 1, 4); + $moviename = substr($moviename, 0, -6); + } + + $query = "SELECT * FROM videos WHERE movie_name = '" . mysqli_real_escape_string($this->conn, $moviename) . "'"; + $result = $this->conn->query($query); + + // insert if not available in db + if (!mysqli_fetch_assoc($result)) { + $genres = -1; + $insert_query = ""; + + // extract other video attributes + $video_attributes = $this->_get_video_attributes($filename); + $duration = 0; + $size = 0; + $width = 0; + + if ($video_attributes) { + $duration = $video_attributes->media->track[0]->Duration; // in seconds + $size = $video_attributes->media->track[0]->FileSize; // in Bytes + $width = $video_attributes->media->track[1]->Width; // width + } + + // extract poster from video + $backpic = shell_exec("$this->ffmpeg -hide_banner -loglevel panic -ss 00:04:00 -i \"../$this->videopath$filename\" -vframes 1 -q:v 2 -f singlejpeg pipe:1 2>/dev/null"); + // convert video to base64 + $backpic64 = 'data:image/jpeg;base64,' . base64_encode($backpic); + + // check if tmdb grabbing is enabled + if ($this->TMDBenabled) { + // search in tmdb api + try { + if (!is_null($dta = $this->tmdb->searchMovie($moviename, $year))) { + $poster = file_get_contents($this->tmdb->picturebase . $dta->poster_path); + + // error handling for download error + if ($poster) { + $poster_base64 = 'data:image/jpeg;base64,' . base64_encode($poster); + + $insert_query = "INSERT INTO videos(movie_name,movie_url,poster,thumbnail,quality,length) + VALUES ('" . mysqli_real_escape_string($this->conn, $moviename) . "', + '" . mysqli_real_escape_string($this->conn, $this->videopath . $filename) . "', + '$backpic64', + '$poster_base64', + '$width', + '$duration')"; + } else { + throw new Exception("faild to load pic"); + } + // store genre ids for parsing later + $genres = $dta->genre_ids; + } else { + // nothing found with tmdb + echo "my moviename: " . $moviename; + + $this->writeLog("nothing found with TMDB! -- $moviename\n"); + throw new Exception("nothing found with TMDB! -- $moviename"); + } + } catch (Exception $e) { + echo $e->getMessage(); + + $insert_query = "INSERT INTO videos(movie_name,movie_url,thumbnail,quality,length) + VALUES ('" . mysqli_real_escape_string($this->conn, $moviename) . "', + '" . mysqli_real_escape_string($this->conn, $this->videopath . $filename) . "', + '$backpic64', + '$width', + '$duration')"; + } + } + + if ($this->conn->query($insert_query) === TRUE) { + echo('successfully added ' . $filename . " to video gravity\n"); + $this->writeLog('successfully added ' . $filename . " to video gravity\n"); + + // add this entry to the default tags + $last_id = $this->conn->insert_id; + + $this->insertSizeTag($width, $last_id); + + // handle tmdb genres here! + if ($genres != -1) { + // transform genre ids in valid names + foreach ($genres as $genreid) { + // check if genre is already a tag in db if not insert it + $tagname = array_column($this->tmdbgenres, 'name', 'id')[$genreid]; + $tagid = $this->tagExists($tagname); + + $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,$tagid)"; + if ($this->conn->query($query) !== TRUE) { + echo "failed to add $genreid tag here.\n"; + $this->writeLog("failed to add $genreid tag here.\n"); + } + } + } + + + $this->added++; + $this->all++; + } else { + echo('errored item: ' . $filename . "\n"); + $this->writeLog('errored item: ' . $filename . "\n"); + echo('{"data":"' . $this->conn->error . '"}\n'); + $this->writeLog('{"data":"' . $this->conn->error . '"}\n'); + $this->failed++; + } + } else { + $this->all++; + } + } + + /** + * insert the corresponding videosize tag to a specific videoid + * @param $width int video width + * @param $videoid int id of video + */ + private function insertSizeTag(int $width, int $videoid) { + // full hd + if ($width >= 1900) { + $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,2)"; + if ($this->conn->query($query) !== TRUE) { + echo "failed to add default tag here.\n"; + $this->writeLog("failed to add default tag here.\n"); + } + } + + // HD + if ($width >= 1250 && $width < 1900) { + $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,4)"; + if ($this->conn->query($query) !== TRUE) { + echo "failed to add default tag here.\n"; + $this->writeLog("failed to add default tag here.\n"); + } + } + + // SD + if ($width < 1250 && $width > 0) { + $query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,3)"; + if ($this->conn->query($query) !== TRUE) { + echo "failed to add default tag here.\n"; + $this->writeLog("failed to add default tag here.\n"); + } + } + } + + /** + * get all videoinfos of a video file + * + * @param $video string name including extension + * @return object all infos as object + */ + private function _get_video_attributes(string $video) { + $command = "mediainfo \"../$this->videopath$video\" --Output=JSON"; + $output = shell_exec($command); + return json_decode($output); + } + + /** + * write a line to the output log file + * + * @param string $message message to write + */ + public function writeLog(string $message) { + file_put_contents("/tmp/output.log", $message, FILE_APPEND); + flush(); + } + + /** + * ckecks if tag exists -- if not creates it + * @param string $tagname the name of the tag + * @return integer the id of the inserted tag + */ + private function tagExists(string $tagname) { + $query = "SELECT * FROM tags WHERE tag_name='$tagname'"; + + $result = $this->conn->query($query); + if ($result->num_rows == 0) { + // tag does not exist --> create it + $query = "INSERT INTO tags (tag_name) VALUES ('$tagname')"; + if ($this->conn->query($query) !== TRUE) { + echo "failed to create $tagname tag in database\n"; + $this->writeLog("failed to create $tagname tag in database\n"); + } + return $this->conn->insert_id; + } else { + return $result->fetch_assoc()['tag_id']; + } + } + + /** + * cleans up the video gravity and removes non existent videos + */ + public function cleanUpGravity() { + // auto cleanup db entries + $query = "SELECT COUNT(*) as count FROM videos"; + $result = $this->conn->query($query); + $r = mysqli_fetch_assoc($result); + + if ($this->all < $r['count']) { + echo "\n\nshould be in gravity: " . $this->all . "\n"; + $this->writeLog("should be in gravity: " . $this->all . "\n"); + echo "really in gravity: " . $r['count'] . "\n"; + $this->writeLog("really in gravity: " . $r['count'] . "\n"); + echo "cleaning up gravity\n"; + $this->writeLog("cleaning up gravity\n"); + + $query = "SELECT movie_id,movie_url FROM videos"; + $result = $this->conn->query($query); + + while ($r = mysqli_fetch_assoc($result)) { + if (!file_exists("../" . $r['movie_url'])) { + $query = "SET foreign_key_checks = 0; DELETE FROM videos WHERE movie_id='" . $r['movie_id'] . "'"; + if ($this->conn->query($query) === TRUE) { + echo('successfully deleted ' . $r['movie_url'] . " from video gravity\n"); + $this->writeLog('successfully deleted ' . $r['movie_url'] . " from video gravity\n"); + $this->deleted++; + } else { + echo "failed to delete " . $r['movie_url'] . " from gravity: " . $this->conn->error . "\n"; + $this->writeLog("failed to delete " . $r['movie_url'] . " from gravity: " . $this->conn->error . "\n"); + } + } + } + } + } +}