lukas/OpenMediaCenter
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup Videoparser and add support for release years in filenames

Browse Source
This commit is contained in:
Lukas Heiligenbrunner 2020-09-23 18:43:50 +00:00
parent b3555efa57
commit 444ef3f074
4 changed files with 356 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

262
api/extractvideopreviews.php
View File

@ -2,266 +2,16 @@
require_once './src/Database.php';
require_once './src/TMDBMovie.php';
require_once './src/SSettings.php';
require_once './src/VideoParser.php';
// allow UTF8 characters
setlocale(LC_ALL, 'en_US.UTF-8');
writeLog("starting extraction!\n");
$vp = new VideoParser();
$vp->writeLog("starting extraction!!\n");
$ffmpeg = 'ffmpeg'; //or: /usr/bin/ffmpeg , or /usr/local/bin/ffmpeg - depends on your installation (type which ffmpeg into a console to find the install path)
$tmdb = new TMDBMovie();
// initial load of all available movie genres
$tmdbgenres = $tmdb->getAllGenres();
$conn = Database::getInstance()->getConnection();
$settings = new SSettings();
$sett = new SSettings();
// load video path from settings
$scandir = "../" . $settings->getVideoPath();
$arr = scandir($scandir);
$all = 0;
$added = 0;
$deleted = 0;
$failed = 0;
foreach ($arr as $elem) {
$ext = pathinfo($elem, PATHINFO_EXTENSION);
if ($ext == "mp4") {
$moviename = substr($elem, 0, -4);
$query = "SELECT * FROM videos WHERE movie_name = '" . mysqli_real_escape_string($conn, $moviename) . "'";
$result = $conn->query($query);
// insert if not available in db
if (!mysqli_fetch_assoc($result)) {
$genres = -1;
$poster = -1; // initially disable poster supp
// extract poster from video
$pic = shell_exec("ffmpeg -hide_banner -loglevel panic -ss 00:04:00 -i \"../videos/prn/$elem\" -vframes 1 -q:v 2 -f singlejpeg pipe:1 2>/dev/null");
// check if tmdb grabbing is enabled
if ($settings->isTMDBGrabbingEnabled()) {
// search in tmdb api
if (!is_null($dta = $tmdb->searchMovie($moviename))) {
$poster = file_get_contents($tmdb->picturebase . $dta->poster_path);
// error handling for download error
if (!$poster) {
echo "Failed to load Picture from TMDB! \n";
}
// store genre ids for parsing later
$genres = $dta->genre_ids;
} else {
// nothing found with tmdb
echo "nothing found with TMDB!\n";
writeLog("nothing found with TMDB!\n");
}
}
// convert video to base64
$image = 'data:image/jpeg;base64,' . base64_encode($pic);
// extract other video attributes
$video_attributes = _get_video_attributes($elem);
$duration = 0;
$size = 0;
$width = 0;
if ($video_attributes) {
$duration = $video_attributes->media->track[0]->Duration; // in seconds
$size = $video_attributes->media->track[0]->FileSize; // in Bytes
$width = $video_attributes->media->track[1]->Width; // width
}
if (!$poster) {
$poster_base64 = 'data:image/jpeg;base64,' . base64_encode($poster);
$query = "INSERT INTO videos(movie_name,movie_url,poster,thumbnail,quality,length)
VALUES ('" . mysqli_real_escape_string($conn, $moviename) . "',
'" . mysqli_real_escape_string($conn, 'videos/prn/' . $elem) . "',
'$poster_base64',
'$image',
'$width',
'$duration')";
} else {
$query = "INSERT INTO videos(movie_name,movie_url,thumbnail,quality,length)
VALUES ('" . mysqli_real_escape_string($conn, $moviename) . "',
'" . mysqli_real_escape_string($conn, 'videos/prn/' . $elem) . "',
'$image',
'$width',
'$duration')";
}
if ($conn->query($query) === TRUE) {
echo('successfully added ' . $elem . " to video gravity\n");
writeLog('successfully added ' . $elem . " to video gravity\n");
// add this entry to the default tags
$last_id = $conn->insert_id;
// full hd
if ($width >= 1900) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,2)";
if ($conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
writeLog("failed to add default tag here.\n");
}
}
// HD
if ($width >= 1250 && $width < 1900) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,4)";
if ($conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
writeLog("failed to add default tag here.\n");
}
}
// SD
if ($width < 1250 && $width > 0) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,3)";
if ($conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
writeLog("failed to add default tag here.\n");
}
}
// handle tmdb genres here!
if ($genres != -1) {
// transform genre ids in valid names
foreach ($genres as $genreid) {
// check if genre is already a tag in db if not insert it
$tagname = array_column($tmdbgenres, 'name', 'id')[$genreid];
$tagid = tagExists($tagname);
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,$tagid)";
if ($conn->query($query) !== TRUE) {
echo "failed to add $genreid tag here.\n";
writeLog("failed to add $genreid tag here.\n");
}
}
}
$added++;
$all++;
} else {
echo('errored item: ' . $elem . "\n");
writeLog('errored item: ' . $elem . "\n");
echo('{"data":"' . $conn->error . '"}\n');
writeLog('{"data":"' . $conn->error . '"}\n');
$failed++;
}
} else {
$all++;
}
} else {
echo($elem . " does not contain a .mp4 extension! - skipping \n");
writeLog($elem . " does not contain a .mp4 extension! - skipping \n");
}
}
// auto cleanup db entries
$query = "SELECT COUNT(*) as count FROM videos";
$result = $conn->query($query);
$r = mysqli_fetch_assoc($result);
if ($all < $r['count']) {
echo "should be in gravity: " . $all . "\n";
writeLog("should be in gravity: " . $all . "\n");
echo "really in gravity: " . $r['count'] . "\n";
writeLog("really in gravity: " . $r['count'] . "\n");
echo "cleaning up gravity\n";
writeLog("cleaning up gravity\n");
$query = "SELECT movie_id,movie_url FROM videos";
$result = $conn->query($query);
while ($r = mysqli_fetch_assoc($result)) {
if (!file_exists("../" . $r['movie_url'])) {
$query = "SET foreign_key_checks = 0; DELETE FROM videos WHERE movie_id='" . $r['movie_id'] . "'";
if ($conn->query($query) === TRUE) {
echo('successfully deleted ' . $r['movie_url'] . " from video gravity\n");
writeLog('successfully deleted ' . $r['movie_url'] . " from video gravity\n");
$deleted++;
} else {
echo "failed to delete " . $r['movie_url'] . " from gravity: " . $conn->error . "\n";
writeLog("failed to delete " . $r['movie_url'] . " from gravity: " . $conn->error . "\n");
}
}
}
}
// calculate size of databse here
$size = -1;
$query = "SELECT table_schema AS \"Database\",
ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS \"Size\"
FROM information_schema.TABLES
WHERE TABLE_SCHEMA='" . Database::getInstance()->getDatabaseName() . "'
GROUP BY table_schema;";
$result = $conn->query($query);
if ($result->num_rows == 1) {
$row = $result->fetch_assoc();
$size = $row["Size"];
}
echo "Total gravity: " . $all . "\n";
writeLog("Total gravity: " . $all . "\n");
echo "Size of Databse is: " . $size . "MB\n";
writeLog("Size of Databse is: " . $size . "MB\n");
echo "added in this run: " . $added . "\n";
writeLog("added in this run: " . $added . "\n");
echo "deleted in this run: " . $deleted . "\n";
writeLog("deleted in this run: " . $deleted . "\n");
echo "errored in this run: " . $failed . "\n";
writeLog("errored in this run: " . $failed . "\n");
writeLog("-42"); // terminating characters to stop webui requesting infos
/**
* get all videoinfos of a video file
*
* @param $video string name including extension
* @return object all infos as object
*/
function _get_video_attributes($video) {
$command = "mediainfo \"../videos/prn/$video\" --Output=JSON";
$output = shell_exec($command);
return json_decode($output);
}
/**
* write a line to the output log file
*
* @param string $message message to write
*/
function writeLog(string $message) {
file_put_contents("/tmp/output.log", $message, FILE_APPEND);
flush();
}
/**
* ckecks if tag exists -- if not creates it
* @param string $tagname the name of the tag
* @return integer the id of the inserted tag
*/
function tagExists(string $tagname) {
global $conn;
$query = "SELECT * FROM tags WHERE tag_name='$tagname'";
$result = $conn->query($query);
if ($result->num_rows == 0) {
// tag does not exist --> create it
$query = "INSERT INTO tags (tag_name) VALUES ('$tagname')";
if ($conn->query($query) !== TRUE) {
echo "failed to create $tagname tag in database\n";
writeLog("failed to create $tagname tag in database\n");
}
return $conn->insert_id;
} else {
return $result->fetch_assoc()['tag_id'];
}
}
$scandir = "../" . $sett->getVideoPath();
$vp->extractVideos($scandir);

2
api/src/SSettings.php
View File

@ -32,7 +32,7 @@ class SSettings {
* @return bool isenabled?
*/
public function isTMDBGrabbingEnabled(): bool {
$query = "SELECT TMDB_grabbing from settings";
$query = "SELECT TMDB_grabbing from settings WHERE 1";
$result = $this->database->getConnection()->query($query);
if (!$result) {

17
api/src/TMDBMovie.php
View File

@ -15,12 +15,25 @@ class TMDBMovie {
* @param string $moviename moviename
* @return object movie object or null if not found
*/
public function searchMovie(string $moviename) {
public function searchMovie(string $moviename, string $year = null) {
$reply = json_decode(file_get_contents($this->baseurl . "search/movie?api_key=" . $this->apikey . "&query=" . urlencode($moviename)));
if ($reply->total_results == 0) {
// no results found
// todo maybe parse first pictures somehow
return null;
} elseif ($year != null) {
// if year is defined check year
$regex = '/[0-9]{4}?/'; // matches year of string
for ($i = 0; $i < count($reply->results); $i++) {
$releasedate = $reply->results[$i]->release_date;
preg_match($regex, $releasedate, $matches);
if (count($matches) > 0) {
$curryear = $matches[0];
if ($curryear == $year)
return $reply->results[$i];
}
}
} else {
return $reply->results[0];
}

334
api/src/VideoParser.php Normal file
View File

@ -0,0 +1,334 @@
<?php
require_once './src/Database.php';
require_once './src/TMDBMovie.php';
require_once './src/SSettings.php';
/**
* Class VideoParser
* handling the parsing of all videos of a folder and adding
* all videos with tags and thumbnails to the database
*/
class VideoParser {
/// ffmpeg installation binary
private string $ffmpeg = 'ffmpeg';
private TMDBMovie $tmdb;
/// initial load of all available movie genres
private array $tmdbgenres;
private string $videopath;
/// db connection instance
private mysqli $conn;
/// settings object instance
private SSettings $settings;
private bool $TMDBenabled;
/// videos added in this run
private int $added = 0;
/// all videos in this run
private int $all = 0;
/// failed videos in this run
private int $failed = 0;
/// deleted videos in this run
private int $deleted = 0;
/**
* VideoParser constructor.
*/
public function __construct() {
$this->tmdb = new TMDBMovie();
$this->tmdbgenres = $this->tmdb->getAllGenres();
$this->conn = Database::getInstance()->getConnection();
$this->settings = new SSettings();
$this->TMDBenabled = $this->settings->isTMDBGrabbingEnabled();
$this->videopath = $this->settings->getVideoPath();
}
/**
* searches a folder for mp4 videos and adds them to video gravity
* @param $foldername string the folder where to search (relative to the webserver root)
*/
public function extractVideos(string $foldername) {
echo("TMDB grabbing is " . ($this->TMDBenabled ? "" : "not") . " enabled \n");
$arr = scandir($foldername);
foreach ($arr as $elem) {
if($elem == '.' || $elem == '..') continue;
$ext = pathinfo($elem, PATHINFO_EXTENSION);
if ($ext == "mp4") {
$this->processVideo($elem);
} else {
echo($elem . " does not contain a valid .mp4 extension! - skipping \n");
$this->writeLog($elem . " does not contain a valid .mp4 extension! - skipping \n");
}
}
// cleanup gravity
$this->cleanUpGravity();
// calculate size of databse here
$size = -1;
$query = "SELECT table_schema AS \"Database\",
ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS \"Size\"
FROM information_schema.TABLES
WHERE TABLE_SCHEMA='" . Database::getInstance()->getDatabaseName() . "'
GROUP BY table_schema;";
$result = $this->conn->query($query);
if ($result->num_rows == 1) {
$row = $result->fetch_assoc();
$size = $row["Size"];
}
echo "Total gravity: " . $this->all . "\n";
$this->writeLog("Total gravity: " . $this->all . "\n");
echo "Size of Databse is: " . $size . "MB\n";
$this->writeLog("Size of Databse is: " . $size . "MB\n");
echo "added in this run: " . $this->added . "\n";
$this->writeLog("added in this run: " . $this->added . "\n");
echo "deleted in this run: " . $this->deleted . "\n";
$this->writeLog("deleted in this run: " . $this->deleted . "\n");
echo "errored in this run: " . $this->failed . "\n";
$this->writeLog("errored in this run: " . $this->failed . "\n");
$this->writeLog("-42"); // terminating characters to stop webui requesting infos
}
/**
* processes one mp4 video, extracts tags and adds it to the database
* @param $filename string filename of the video to process
*/
private function processVideo(string $filename) {
$moviename = substr($filename, 0, -4);
$regex = '/\([0-9]{4}?\)/'; //match year pattern
preg_match($regex, $moviename, $matches);
preg_replace($regex, '', $moviename);
$year = null;
if (count($matches) > 0) {
$year = substr($matches[count($matches) - 1], 1, 4);
$moviename = substr($moviename, 0, -6);
}
$query = "SELECT * FROM videos WHERE movie_name = '" . mysqli_real_escape_string($this->conn, $moviename) . "'";
$result = $this->conn->query($query);
// insert if not available in db
if (!mysqli_fetch_assoc($result)) {
$genres = -1;
$insert_query = "";
// extract other video attributes
$video_attributes = $this->_get_video_attributes($filename);
$duration = 0;
$size = 0;
$width = 0;
if ($video_attributes) {
$duration = $video_attributes->media->track[0]->Duration; // in seconds
$size = $video_attributes->media->track[0]->FileSize; // in Bytes
$width = $video_attributes->media->track[1]->Width; // width
}
// extract poster from video
$backpic = shell_exec("$this->ffmpeg -hide_banner -loglevel panic -ss 00:04:00 -i \"../$this->videopath$filename\" -vframes 1 -q:v 2 -f singlejpeg pipe:1 2>/dev/null");
// convert video to base64
$backpic64 = 'data:image/jpeg;base64,' . base64_encode($backpic);
// check if tmdb grabbing is enabled
if ($this->TMDBenabled) {
// search in tmdb api
try {
if (!is_null($dta = $this->tmdb->searchMovie($moviename, $year))) {
$poster = file_get_contents($this->tmdb->picturebase . $dta->poster_path);
// error handling for download error
if ($poster) {
$poster_base64 = 'data:image/jpeg;base64,' . base64_encode($poster);
$insert_query = "INSERT INTO videos(movie_name,movie_url,poster,thumbnail,quality,length)
VALUES ('" . mysqli_real_escape_string($this->conn, $moviename) . "',
'" . mysqli_real_escape_string($this->conn, $this->videopath . $filename) . "',
'$backpic64',
'$poster_base64',
'$width',
'$duration')";
} else {
throw new Exception("faild to load pic");
}
// store genre ids for parsing later
$genres = $dta->genre_ids;
} else {
// nothing found with tmdb
echo "my moviename: " . $moviename;
$this->writeLog("nothing found with TMDB! -- $moviename\n");
throw new Exception("nothing found with TMDB! -- $moviename");
}
} catch (Exception $e) {
echo $e->getMessage();
$insert_query = "INSERT INTO videos(movie_name,movie_url,thumbnail,quality,length)
VALUES ('" . mysqli_real_escape_string($this->conn, $moviename) . "',
'" . mysqli_real_escape_string($this->conn, $this->videopath . $filename) . "',
'$backpic64',
'$width',
'$duration')";
}
}
if ($this->conn->query($insert_query) === TRUE) {
echo('successfully added ' . $filename . " to video gravity\n");
$this->writeLog('successfully added ' . $filename . " to video gravity\n");
// add this entry to the default tags
$last_id = $this->conn->insert_id;
$this->insertSizeTag($width, $last_id);
// handle tmdb genres here!
if ($genres != -1) {
// transform genre ids in valid names
foreach ($genres as $genreid) {
// check if genre is already a tag in db if not insert it
$tagname = array_column($this->tmdbgenres, 'name', 'id')[$genreid];
$tagid = $this->tagExists($tagname);
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($last_id,$tagid)";
if ($this->conn->query($query) !== TRUE) {
echo "failed to add $genreid tag here.\n";
$this->writeLog("failed to add $genreid tag here.\n");
}
}
}
$this->added++;
$this->all++;
} else {
echo('errored item: ' . $filename . "\n");
$this->writeLog('errored item: ' . $filename . "\n");
echo('{"data":"' . $this->conn->error . '"}\n');
$this->writeLog('{"data":"' . $this->conn->error . '"}\n');
$this->failed++;
}
} else {
$this->all++;
}
}
/**
* insert the corresponding videosize tag to a specific videoid
* @param $width int video width
* @param $videoid int id of video
*/
private function insertSizeTag(int $width, int $videoid) {
// full hd
if ($width >= 1900) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,2)";
if ($this->conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
$this->writeLog("failed to add default tag here.\n");
}
}
// HD
if ($width >= 1250 && $width < 1900) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,4)";
if ($this->conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
$this->writeLog("failed to add default tag here.\n");
}
}
// SD
if ($width < 1250 && $width > 0) {
$query = "INSERT INTO video_tags(video_id,tag_id) VALUES ($videoid,3)";
if ($this->conn->query($query) !== TRUE) {
echo "failed to add default tag here.\n";
$this->writeLog("failed to add default tag here.\n");
}
}
}
/**
* get all videoinfos of a video file
*
* @param $video string name including extension
* @return object all infos as object
*/
private function _get_video_attributes(string $video) {
$command = "mediainfo \"../$this->videopath$video\" --Output=JSON";
$output = shell_exec($command);
return json_decode($output);
}
/**
* write a line to the output log file
*
* @param string $message message to write
*/
public function writeLog(string $message) {
file_put_contents("/tmp/output.log", $message, FILE_APPEND);
flush();
}
/**
* ckecks if tag exists -- if not creates it
* @param string $tagname the name of the tag
* @return integer the id of the inserted tag
*/
private function tagExists(string $tagname) {
$query = "SELECT * FROM tags WHERE tag_name='$tagname'";
$result = $this->conn->query($query);
if ($result->num_rows == 0) {
// tag does not exist --> create it
$query = "INSERT INTO tags (tag_name) VALUES ('$tagname')";
if ($this->conn->query($query) !== TRUE) {
echo "failed to create $tagname tag in database\n";
$this->writeLog("failed to create $tagname tag in database\n");
}
return $this->conn->insert_id;
} else {
return $result->fetch_assoc()['tag_id'];
}
}
/**
* cleans up the video gravity and removes non existent videos
*/
public function cleanUpGravity() {
// auto cleanup db entries
$query = "SELECT COUNT(*) as count FROM videos";
$result = $this->conn->query($query);
$r = mysqli_fetch_assoc($result);
if ($this->all < $r['count']) {
echo "\n\nshould be in gravity: " . $this->all . "\n";
$this->writeLog("should be in gravity: " . $this->all . "\n");
echo "really in gravity: " . $r['count'] . "\n";
$this->writeLog("really in gravity: " . $r['count'] . "\n");
echo "cleaning up gravity\n";
$this->writeLog("cleaning up gravity\n");
$query = "SELECT movie_id,movie_url FROM videos";
$result = $this->conn->query($query);
while ($r = mysqli_fetch_assoc($result)) {
if (!file_exists("../" . $r['movie_url'])) {
$query = "SET foreign_key_checks = 0; DELETE FROM videos WHERE movie_id='" . $r['movie_id'] . "'";
if ($this->conn->query($query) === TRUE) {
echo('successfully deleted ' . $r['movie_url'] . " from video gravity\n");
$this->writeLog('successfully deleted ' . $r['movie_url'] . " from video gravity\n");
$this->deleted++;
} else {
echo "failed to delete " . $r['movie_url'] . " from gravity: " . $this->conn->error . "\n";
$this->writeLog("failed to delete " . $r['movie_url'] . " from gravity: " . $this->conn->error . "\n");
}
}
}
}
}
}