implement full load of videos and startdata
modify api where necessary
This commit is contained in:
1
apiGo/videoparser/CleanUp.go
Normal file
1
apiGo/videoparser/CleanUp.go
Normal file
@ -0,0 +1 @@
|
||||
package videoparser
|
||||
294
apiGo/videoparser/ReIndex.go
Normal file
294
apiGo/videoparser/ReIndex.go
Normal file
@ -0,0 +1,294 @@
|
||||
package videoparser
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"openmediacenter/apiGo/api/types"
|
||||
"openmediacenter/apiGo/database"
|
||||
"openmediacenter/apiGo/videoparser/tmdb"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
var mSettings types.SettingsType
|
||||
var mExtDepsAvailable *ExtDependencySupport
|
||||
|
||||
// default Tag ids
|
||||
const (
|
||||
FullHd = 2
|
||||
Hd = 4
|
||||
LowQuality = 3
|
||||
)
|
||||
|
||||
type ExtDependencySupport struct {
|
||||
FFMpeg bool
|
||||
MediaInfo bool
|
||||
}
|
||||
|
||||
type VideoAttributes struct {
|
||||
Duration float32
|
||||
FileSize uint
|
||||
Width uint
|
||||
}
|
||||
|
||||
func ReIndexVideos(path []string, sett types.SettingsType) {
|
||||
mSettings = sett
|
||||
// check if the extern dependencies are available
|
||||
mExtDepsAvailable = checkExtDependencySupport()
|
||||
fmt.Printf("FFMPEG support: %t\n", mExtDepsAvailable.FFMpeg)
|
||||
fmt.Printf("MediaInfo support: %t\n", mExtDepsAvailable.MediaInfo)
|
||||
|
||||
for _, s := range path {
|
||||
processVideo(s)
|
||||
}
|
||||
|
||||
AppendMessageBuffer("reindex finished successfully!")
|
||||
|
||||
contentAvailable = false
|
||||
fmt.Println("Reindexing finished!")
|
||||
}
|
||||
|
||||
func processVideo(fileNameOrig string) {
|
||||
fmt.Printf("Processing %s video-", fileNameOrig)
|
||||
|
||||
// match the file extension
|
||||
r, _ := regexp.Compile(`\.[a-zA-Z0-9]+$`)
|
||||
fileName := r.ReplaceAllString(fileNameOrig, "")
|
||||
|
||||
year, fileName := matchYear(fileName)
|
||||
|
||||
// now we should look if this video already exists in db
|
||||
query := "SELECT * FROM videos WHERE movie_name = ?"
|
||||
err := database.QueryRow(query, fileName).Scan()
|
||||
if err == sql.ErrNoRows {
|
||||
fmt.Printf("The Video %s does't exist! Adding it to database.\n", fileName)
|
||||
|
||||
addVideo(fileName, fileNameOrig, year)
|
||||
} else {
|
||||
fmt.Println(" :existing!")
|
||||
}
|
||||
}
|
||||
|
||||
// add a video to the database
|
||||
func addVideo(videoName string, fileName string, year int) {
|
||||
var ppic *string
|
||||
var poster *string
|
||||
var tmdbData *tmdb.VideoTMDB
|
||||
var err error
|
||||
|
||||
// initialize defaults
|
||||
vidAtr := &VideoAttributes{
|
||||
Duration: 0,
|
||||
FileSize: 0,
|
||||
Width: 0,
|
||||
}
|
||||
|
||||
if mExtDepsAvailable.FFMpeg {
|
||||
ppic, err = parseFFmpegPic(fileName)
|
||||
if err != nil {
|
||||
fmt.Printf("FFmpeg error occured: %s\n", err.Error())
|
||||
} else {
|
||||
fmt.Println("successfully extracted thumbnail!!")
|
||||
}
|
||||
}
|
||||
|
||||
if mExtDepsAvailable.MediaInfo {
|
||||
atr := getVideoAttributes(fileName)
|
||||
if atr != nil {
|
||||
vidAtr = atr
|
||||
}
|
||||
}
|
||||
|
||||
// if TMDB grabbing is enabled serach in api for video...
|
||||
if mSettings.TMDBGrabbing {
|
||||
tmdbData = tmdb.SearchVideo(videoName, year)
|
||||
if tmdbData != nil {
|
||||
// reassign parsed pic as poster
|
||||
poster = ppic
|
||||
// and tmdb pic as thumbnail
|
||||
ppic = &tmdbData.Thumbnail
|
||||
}
|
||||
}
|
||||
|
||||
query := `INSERT INTO videos(movie_name,movie_url,poster,thumbnail,quality,length) VALUES (?,?,?,?,?,?)`
|
||||
err, insertId := database.Insert(query, videoName, fileName, poster, ppic, vidAtr.Width, vidAtr.Duration)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to insert video into db: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// add default tags
|
||||
if vidAtr.Width != 0 {
|
||||
insertSizeTag(vidAtr.Width, uint(insertId))
|
||||
}
|
||||
|
||||
// add tmdb tags
|
||||
if mSettings.TMDBGrabbing && tmdbData != nil {
|
||||
insertTMDBTags(tmdbData.GenreIds, insertId)
|
||||
}
|
||||
|
||||
AppendMessageBuffer(fmt.Sprintf("%s - added!", videoName))
|
||||
}
|
||||
|
||||
func matchYear(fileName string) (int, string) {
|
||||
r, _ := regexp.Compile(`\([0-9]{4}?\)`)
|
||||
years := r.FindAllString(fileName, -1)
|
||||
if len(years) == 0 {
|
||||
return -1, fileName
|
||||
}
|
||||
|
||||
year, err := strconv.Atoi(years[len(years)-1])
|
||||
|
||||
if err != nil {
|
||||
return -1, fileName
|
||||
}
|
||||
|
||||
// cut out year from filename
|
||||
return year, r.ReplaceAllString(fileName, "")
|
||||
}
|
||||
|
||||
// parse the thumbail picture from video file
|
||||
func parseFFmpegPic(fileName string) (*string, error) {
|
||||
app := "ffmpeg"
|
||||
|
||||
cmd := exec.Command(app,
|
||||
"-hide_banner",
|
||||
"-loglevel", "panic",
|
||||
"-ss", "00:04:00",
|
||||
"-i", mSettings.VideoPath+fileName,
|
||||
"-vframes", "1",
|
||||
"-q:v", "2",
|
||||
"-f", "singlejpeg",
|
||||
"pipe:1")
|
||||
|
||||
stdout, err := cmd.Output()
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
fmt.Println(string(err.(*exec.ExitError).Stderr))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
backpic64 := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(stdout)
|
||||
|
||||
return &backpic64, nil
|
||||
}
|
||||
|
||||
func getVideoAttributes(fileName string) *VideoAttributes {
|
||||
app := "mediainfo"
|
||||
|
||||
arg0 := mSettings.VideoPath + fileName
|
||||
arg1 := "--Output=JSON"
|
||||
|
||||
cmd := exec.Command(app, arg1, "-f", arg0)
|
||||
stdout, err := cmd.Output()
|
||||
|
||||
var t struct {
|
||||
Media struct {
|
||||
Track []struct {
|
||||
Duration string
|
||||
FileSize string
|
||||
Width string
|
||||
}
|
||||
}
|
||||
}
|
||||
err = json.Unmarshal(stdout, &t)
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
duration, err := strconv.ParseFloat(t.Media.Track[0].Duration, 32)
|
||||
filesize, err := strconv.Atoi(t.Media.Track[0].FileSize)
|
||||
width, err := strconv.Atoi(t.Media.Track[1].Width)
|
||||
|
||||
ret := VideoAttributes{
|
||||
Duration: float32(duration),
|
||||
FileSize: uint(filesize),
|
||||
Width: uint(width),
|
||||
}
|
||||
|
||||
return &ret
|
||||
}
|
||||
|
||||
func AppendMessageBuffer(message string) {
|
||||
messageBuffer = append(messageBuffer, message)
|
||||
}
|
||||
|
||||
// ext dependency support check
|
||||
func checkExtDependencySupport() *ExtDependencySupport {
|
||||
var extDepsAvailable ExtDependencySupport
|
||||
|
||||
extDepsAvailable.FFMpeg = commandExists("ffmpeg")
|
||||
extDepsAvailable.MediaInfo = commandExists("mediainfo")
|
||||
|
||||
return &extDepsAvailable
|
||||
}
|
||||
|
||||
// check if a specific system command is available
|
||||
func commandExists(cmd string) bool {
|
||||
_, err := exec.LookPath(cmd)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// insert the default size tags to corresponding video
|
||||
func insertSizeTag(width uint, videoId uint) {
|
||||
var tagType uint
|
||||
|
||||
if width >= 1080 {
|
||||
tagType = FullHd
|
||||
} else if width >= 720 {
|
||||
tagType = Hd
|
||||
} else {
|
||||
tagType = LowQuality
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("INSERT INTO video_tags(video_id,tag_id) VALUES (%d,%d)", videoId, tagType)
|
||||
err := database.Edit(query)
|
||||
if err != nil {
|
||||
fmt.Printf("Eror occured while adding default Tag: %s\n", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// insert id array of tmdb geners to database
|
||||
func insertTMDBTags(ids []int, videoId int64) {
|
||||
genres := tmdb.GetGenres()
|
||||
|
||||
for _, id := range ids {
|
||||
var idGenre *tmdb.TMDBGenre
|
||||
for _, genre := range *genres {
|
||||
if genre.Id == id {
|
||||
idGenre = &genre
|
||||
break
|
||||
}
|
||||
}
|
||||
// skip tag if name couldn't be found
|
||||
if idGenre == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// now we check if the tag we want to add already exists
|
||||
tagId := createTagToDB(idGenre.Name)
|
||||
|
||||
// now we add the tag
|
||||
query := fmt.Sprintf("INSERT INTO video_tags(video_id,tag_id) VALUES (%d,%d)", videoId, tagId)
|
||||
_ = database.Edit(query)
|
||||
}
|
||||
}
|
||||
|
||||
// returns id of tag or creates it if not existing
|
||||
func createTagToDB(tagName string) int64 {
|
||||
query := fmt.Sprintf("SELECT tag_id FROM tags WHERE tag_name = %s", tagName)
|
||||
var id int64
|
||||
err := database.QueryRow(query).Scan(&id)
|
||||
if err == sql.ErrNoRows {
|
||||
// tag doesn't exist -- add it
|
||||
query = fmt.Sprintf("INSERT INTO tags (tag_name) VALUES (%s)", tagName)
|
||||
err, id = database.Insert(query)
|
||||
}
|
||||
return id
|
||||
}
|
||||
70
apiGo/videoparser/VideoParser.go
Normal file
70
apiGo/videoparser/VideoParser.go
Normal file
@ -0,0 +1,70 @@
|
||||
package videoparser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"openmediacenter/apiGo/database"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var messageBuffer []string
|
||||
var contentAvailable = false
|
||||
|
||||
type StatusMessage struct {
|
||||
Messages []string
|
||||
ContentAvailable bool
|
||||
}
|
||||
|
||||
func StartReindex() bool {
|
||||
messageBuffer = []string{}
|
||||
contentAvailable = true
|
||||
|
||||
fmt.Println("starting reindex..")
|
||||
|
||||
mSettings := database.GetSettings()
|
||||
// add the path prefix to videopath
|
||||
mSettings.VideoPath = mSettings.PathPrefix + mSettings.VideoPath
|
||||
|
||||
// check if path even exists
|
||||
if _, err := os.Stat(mSettings.VideoPath); os.IsNotExist(err) {
|
||||
fmt.Println("Reindex path doesn't exist!")
|
||||
return false
|
||||
}
|
||||
|
||||
var files []string
|
||||
err := filepath.Walk(mSettings.VideoPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.IsDir() && strings.HasSuffix(info.Name(), ".mp4") {
|
||||
files = append(files, info.Name())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
}
|
||||
// start reindex process
|
||||
AppendMessageBuffer("Starting Reindexing!")
|
||||
go ReIndexVideos(files, mSettings)
|
||||
return true
|
||||
}
|
||||
|
||||
func GetStatusMessage() *StatusMessage {
|
||||
msg := StatusMessage{
|
||||
Messages: messageBuffer,
|
||||
ContentAvailable: contentAvailable,
|
||||
}
|
||||
|
||||
messageBuffer = []string{}
|
||||
|
||||
return &msg
|
||||
}
|
||||
|
||||
func StartCleanup() {
|
||||
// todo start cleanup
|
||||
}
|
||||
144
apiGo/videoparser/tmdb/TMDBApi.go
Normal file
144
apiGo/videoparser/tmdb/TMDBApi.go
Normal file
@ -0,0 +1,144 @@
|
||||
package tmdb
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
const apiKey = "9fd90530b11447f5646f8e6fb4733fb4"
|
||||
const baseUrl = "https://api.themoviedb.org/3/"
|
||||
const pictureBase = "https://image.tmdb.org/t/p/w500"
|
||||
|
||||
type VideoTMDB struct {
|
||||
Thumbnail string
|
||||
Overview string
|
||||
Title string
|
||||
GenreIds []int
|
||||
}
|
||||
|
||||
type tmdbVidResult struct {
|
||||
Poster_path string
|
||||
Adult bool
|
||||
Overview string
|
||||
Release_date string
|
||||
Genre_ids []int
|
||||
Id int
|
||||
Original_title string
|
||||
Original_language string
|
||||
Title string
|
||||
Backdrop_path string
|
||||
Popularity int
|
||||
Vote_count int
|
||||
Video bool
|
||||
Vote_average int
|
||||
}
|
||||
|
||||
type TMDBGenre struct {
|
||||
Id int
|
||||
Name string
|
||||
}
|
||||
|
||||
func SearchVideo(MovieName string, year int) *VideoTMDB {
|
||||
url := fmt.Sprintf("%ssearch/movie?api_key=%s&query=%s", baseUrl, apiKey, MovieName)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
var t struct {
|
||||
Results []tmdbVidResult
|
||||
}
|
||||
err = json.Unmarshal(body, &t)
|
||||
|
||||
fmt.Println(len(t.Results))
|
||||
|
||||
var tmdbVid tmdbVidResult
|
||||
if year != -1 {
|
||||
for _, result := range t.Results {
|
||||
r, _ := regexp.Compile(fmt.Sprintf(`^%d-[0-9]{2}?-[0-9]{2}?$`, year))
|
||||
if r.MatchString(result.Release_date) {
|
||||
tmdbVid = result
|
||||
// continue parsing
|
||||
goto cont
|
||||
}
|
||||
}
|
||||
// if there is no match use first one
|
||||
tmdbVid = t.Results[0]
|
||||
} else {
|
||||
tmdbVid = t.Results[0]
|
||||
}
|
||||
|
||||
// continue label
|
||||
cont:
|
||||
|
||||
thumbnail := fetchPoster(tmdbVid)
|
||||
|
||||
result := VideoTMDB{
|
||||
Thumbnail: *thumbnail,
|
||||
Overview: tmdbVid.Overview,
|
||||
Title: tmdbVid.Title,
|
||||
GenreIds: tmdbVid.Genre_ids,
|
||||
}
|
||||
|
||||
return &result
|
||||
}
|
||||
|
||||
func fetchPoster(vid tmdbVidResult) *string {
|
||||
url := fmt.Sprintf("%s%s", pictureBase, vid.Poster_path)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
backpic64 := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(body)
|
||||
return &backpic64
|
||||
}
|
||||
|
||||
var tmdbGenres *[]TMDBGenre
|
||||
|
||||
func fetchGenres() *[]TMDBGenre {
|
||||
url := fmt.Sprintf("%sgenre/movie/list?api_key=%s", baseUrl, apiKey)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
var t []TMDBGenre
|
||||
err = json.Unmarshal(body, &t)
|
||||
|
||||
return &t
|
||||
}
|
||||
|
||||
func GetGenres() *[]TMDBGenre {
|
||||
// if generes are nil fetch them once
|
||||
if tmdbGenres == nil {
|
||||
tmdbGenres = fetchGenres()
|
||||
}
|
||||
return tmdbGenres
|
||||
}
|
||||
Reference in New Issue
Block a user