From f7a0d8fa0784afc4be29b9973bdeb81f295e4e84 Mon Sep 17 00:00:00 2001 From: lukas-heiligenbrunner Date: Thu, 5 May 2022 21:32:35 +0200 Subject: [PATCH] add housekeeping code to delete duplicated tags and videos --- apiGo/api/Tags.go | 2 +- apiGo/housekeeping/HouseKeeping.go | 18 +++++ apiGo/housekeeping/MissingVideoMetaData.go | 5 ++ apiGo/housekeeping/TagDeduplication.go | 86 +++++++++++++++++++++ apiGo/housekeeping/VideoTagDeduplication.go | 37 +++++++++ apiGo/main.go | 10 +++ 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 apiGo/housekeeping/HouseKeeping.go create mode 100644 apiGo/housekeeping/MissingVideoMetaData.go create mode 100644 apiGo/housekeeping/TagDeduplication.go create mode 100644 apiGo/housekeeping/VideoTagDeduplication.go diff --git a/apiGo/api/Tags.go b/apiGo/api/Tags.go index 5c3bfb1..552b515 100644 --- a/apiGo/api/Tags.go +++ b/apiGo/api/Tags.go @@ -78,7 +78,7 @@ func getFromDB() { * @apiSuccess {string} TagName name of the Tag */ api.AddHandler("getAllTags", api.TagNode, api.PermUser, func(context api.Context) { - query := "SELECT tag_id,tag_name from tags" + query := "SELECT tag_id,tag_name from tags ORDER BY tag_name" context.Json(readTagsFromResultset(database.Query(query))) }) } diff --git a/apiGo/housekeeping/HouseKeeping.go b/apiGo/housekeeping/HouseKeeping.go new file mode 100644 index 0000000..dbc453e --- /dev/null +++ b/apiGo/housekeeping/HouseKeeping.go @@ -0,0 +1,18 @@ +package housekeeping + +import "fmt" + +func RunHouseKeepingTasks() { + fmt.Println("Runnint houskeeping tasks!") + + fmt.Println("Deduplicating Tags") + deduplicateTags() + + fmt.Println("Deduplicating Tags assigned to videos") + deduplicateVideoTags() + + fmt.Println("Fix missing video metadata like ratio") + fixMissingMetadata() + + fmt.Println("Finished housekeeping") +} diff --git a/apiGo/housekeeping/MissingVideoMetaData.go b/apiGo/housekeeping/MissingVideoMetaData.go new file mode 100644 index 0000000..3b9a20c --- /dev/null +++ b/apiGo/housekeeping/MissingVideoMetaData.go @@ -0,0 +1,5 @@ +package housekeeping + +func fixMissingMetadata() { + // todo +} diff --git a/apiGo/housekeeping/TagDeduplication.go b/apiGo/housekeeping/TagDeduplication.go new file mode 100644 index 0000000..0493e28 --- /dev/null +++ b/apiGo/housekeeping/TagDeduplication.go @@ -0,0 +1,86 @@ +package housekeeping + +import ( + "fmt" + "openmediacenter/apiGo/database" +) + +func deduplicateTags() { + // find all duplicate tags + + // gives first occurence of duplicate + query := ` +SELECT + tag_name +FROM + tags +GROUP BY tag_name +HAVING COUNT(tag_name) > 1` + rows := database.Query(query) + duplicates := []string{} + if rows != nil { + for rows.Next() { + var id string + err := rows.Scan(&id) + if err != nil { + panic(err.Error()) // proper Error handling instead of panic in your app + } + duplicates = append(duplicates, id) + } + } else { + // nothing to do + return + } + + fmt.Print("deleting duplicate tag ids: ") + fmt.Println(duplicates) + + for _, el := range duplicates { + query := fmt.Sprintf("SELECT tag_id FROM tags WHERE tag_name='%s'", el) + rows := database.Query(query) + ids := []uint32{} + for rows.Next() { + var id uint32 + err := rows.Scan(&id) + if err != nil { + panic(err.Error()) // proper Error handling instead of panic in your app + } + ids = append(ids, id) + } + + // id to copy other data to + mainid := ids[0] + + // ids to copy from + copyids := ids[1:] + + fmt.Printf("Migrating %s\n", el) + migrateTags(mainid, copyids) + } +} + +func migrateTags(destid uint32, sourcids []uint32) { + querytempl := ` +UPDATE video_tags +SET + tag_id = %d +WHERE + tag_id = %d` + + for _, id := range sourcids { + err := database.Edit(fmt.Sprintf(querytempl, destid, id)) + if err != nil { + fmt.Printf("failed to set id from %d to %d\n", id, destid) + return + } + fmt.Printf("Merged %d into %d\n", id, destid) + + // now lets delete this tag + query := fmt.Sprintf(`DELETE FROM tags WHERE tag_id=%d`, id) + err = database.Edit(query) + if err != nil { + fmt.Printf("failed to delete Tag %d", id) + return + } + } +} diff --git a/apiGo/housekeeping/VideoTagDeduplication.go b/apiGo/housekeeping/VideoTagDeduplication.go new file mode 100644 index 0000000..9c8bc3a --- /dev/null +++ b/apiGo/housekeeping/VideoTagDeduplication.go @@ -0,0 +1,37 @@ +package housekeeping + +import ( + "fmt" + "openmediacenter/apiGo/database" +) + +func deduplicateVideoTags() { + // gives first occurence of duplicate + query := ` +SELECT + tag_id, video_id, count(tag_id) +FROM + video_tags +GROUP BY tag_id, video_id +HAVING COUNT(tag_id) > 1` + rows := database.Query(query) + if rows != nil { + for rows.Next() { + var tagid uint32 + var vidid uint32 + var nr uint32 + err := rows.Scan(&tagid, &vidid, &nr) + if err != nil { + panic(err.Error()) // proper Error handling instead of panic in your app + } + + // now lets delete this tag + query := fmt.Sprintf(`DELETE FROM video_tags WHERE tag_id=%d AND video_id=%d LIMIT %d`, tagid, vidid, nr-1) + err = database.Edit(query) + if err != nil { + fmt.Printf("failed to delete Tag %d + vid %d", tagid, vidid) + return + } + } + } +} diff --git a/apiGo/main.go b/apiGo/main.go index f93e28f..58575b3 100644 --- a/apiGo/main.go +++ b/apiGo/main.go @@ -1,11 +1,13 @@ package main import ( + "flag" "fmt" "openmediacenter/apiGo/api" api2 "openmediacenter/apiGo/api/api" "openmediacenter/apiGo/config" "openmediacenter/apiGo/database" + "openmediacenter/apiGo/housekeeping" "openmediacenter/apiGo/static" "openmediacenter/apiGo/videoparser" "os" @@ -17,6 +19,8 @@ func main() { const port uint16 = 8081 errc := make(chan error, 1) + housekPTr := flag.Bool("HouseKeeping", false, "Run housekeeping tasks") + config.Init() // todo some verbosity logger or sth @@ -29,6 +33,12 @@ func main() { } defer database.Close() + // check if we should run the housekeeping tasks + if *housekPTr { + housekeeping.RunHouseKeepingTasks() + return + } + api.AddHandlers() videoparser.SetupSettingsWebsocket()