add improved db layout with pkg versions

better error handling of api

.gitignore

@@ -3,10 +3,6 @@
 debug/
 target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 

Dockerfile

@@ -30,4 +30,5 @@ RUN pacman -Sc
 # EXPOSE 8080
 
 # Set the entry point or default command to run your application
+WORKDIR /app
 CMD ["untitled"]

README.md

@@ -1,2 +1,17 @@
 # AURCache
 A cache build server for Archlinux AUR packages and serving them
+
+
+## Things still missing
+
+* proper error return to api
+* package updates
+* multiple package versions
+* error checks if requested package does not exist
+* proper logging
+* auto update packages
+* built package version differs from aur pkg version eg. mesa-git
+* implement repo-add in rust
+* cicd
+* build table where all version builds are with stdout 
+* endpoint to get build log

scripts/README.md

@@ -0,0 +1,2 @@
+This is a patched makepkg version to allow being run as root. 
+Especially in containers this makes things a lot easier and shoudln't be a security concern there.

src/api/backend.rs

@@ -1,6 +1,6 @@
 use crate::aur::aur::get_info_by_name;
 use crate::builder::types::Action;
-use crate::db::packages;
+use crate::db::{packages, versions};
 use crate::query_aur;
 use rocket::serde::json::Json;
 use rocket::serde::{Deserialize, Serialize};
@@ -8,12 +8,13 @@ use rocket::State;
 use rocket::{get, post, Route};
 use rocket_okapi::okapi::schemars;
 use rocket_okapi::{openapi, openapi_get_routes, JsonSchema};
-use sea_orm::{ActiveModelTrait, DatabaseConnection, Set};
+use sea_orm::EntityTrait;
-use sea_orm::{DeleteResult, EntityTrait, ModelTrait};
+use sea_orm::{ActiveModelTrait, DatabaseConnection, FromQueryResult, JoinType, QuerySelect, Set};
+use sea_orm::{ColumnTrait, RelationTrait};
 use tokio::sync::broadcast::Sender;
 
 use crate::db::prelude::Packages;
-use crate::repo::repo::remove_pkg;
+use crate::repo::repo::{remove_pkg, remove_version};
 
 #[derive(Serialize, JsonSchema)]
 #[serde(crate = "rocket::serde")]
@@ -42,14 +43,30 @@ async fn search(query: &str) -> Result<Json<Vec<ApiPackage>>, String> {
     }
 }
 
+#[derive(FromQueryResult, Deserialize, JsonSchema, Serialize)]
+#[serde(crate = "rocket::serde")]
+struct ListPackageModel {
+    name: String,
+    count: i32,
+}
+
 #[openapi(tag = "test")]
 #[get("/packages/list")]
 async fn package_list(
     db: &State<DatabaseConnection>,
-) -> Result<Json<Vec<packages::Model>>, String> {
+) -> Result<Json<Vec<ListPackageModel>>, String> {
     let db = db as &DatabaseConnection;
 
-    let all: Vec<packages::Model> = Packages::find().all(db).await.unwrap();
+    let all: Vec<ListPackageModel> = Packages::find()
+        .join_rev(JoinType::InnerJoin, versions::Relation::Packages.def())
+        .select_only()
+        .column_as(versions::Column::Id.count(), "count")
+        .column(packages::Column::Name)
+        .group_by(packages::Column::Name)
+        .into_model::<ListPackageModel>()
+        .all(db)
+        .await
+        .unwrap();
 
     Ok(Json(all))
 }
@@ -68,7 +85,6 @@ async fn package_add(
     tx: &State<Sender<Action>>,
 ) -> Result<(), String> {
     let db = db as &DatabaseConnection;
-
     let pkg_name = &input.name;
 
     let pkg = get_info_by_name(pkg_name)
@@ -77,17 +93,24 @@ async fn package_add(
 
     let new_package = packages::ActiveModel {
         name: Set(pkg_name.clone()),
-        version: Set(pkg.version.clone()),
         ..Default::default()
     };
 
-    let t = new_package.save(db).await.expect("TODO: panic message");
+    let pkt_model = new_package.save(db).await.expect("TODO: panic message");
+
+    let new_version = versions::ActiveModel {
+        version: Set(pkg.version.clone()),
+        package_id: Set(pkt_model.id.clone().unwrap()),
+        ..Default::default()
+    };
+
+    let version_model = new_version.save(db).await.expect("TODO: panic message");
 
     let _ = tx.send(Action::Build(
         pkg.name,
         pkg.version,
         pkg.url_path.unwrap(),
-        t.id.unwrap(),
+        version_model,
     ));
 
     Ok(())
@@ -103,22 +126,23 @@ struct DelBody {
 #[post("/packages/delete", data = "<input>")]
 async fn package_del(db: &State<DatabaseConnection>, input: Json<DelBody>) -> Result<(), String> {
     let db = db as &DatabaseConnection;
-    let pkg_id = &input.id;
+    let pkg_id = input.id.clone();
 
-    let pkg = Packages::find_by_id(*pkg_id)
+    remove_pkg(db, pkg_id).await.map_err(|e| e.to_string())?;
-        .one(db)
-        .await
-        .unwrap()
-        .unwrap();
 
-    // remove folders
+    Ok(())
-    remove_pkg(pkg.name.to_string(), pkg.version.to_string()).await;
+}
+
+#[openapi(tag = "test")]
+#[post("/versions/delete/<id>")]
+async fn version_del(db: &State<DatabaseConnection>, id: i32) -> Result<(), String> {
+    let db = db as &DatabaseConnection;
+
+    remove_version(db, id).await.map_err(|e| e.to_string())?;
 
-    // remove package db entry
-    let res: DeleteResult = pkg.delete(db).await.unwrap();
     Ok(())
 }
 
 pub fn build_api() -> Vec<Route> {
-    openapi_get_routes![search, package_list, package_add, package_del]
+    openapi_get_routes![search, package_list, package_add, package_del, version_del]
 }

src/api/mod.rs

@@ -1,2 +1 @@
 pub mod backend;
-pub mod repository;

src/api/repository.rs

@@ -1,5 +0,0 @@
-use rocket::fs::FileServer;
-
-pub fn build_api() -> FileServer {
-    FileServer::from("./repo")
-}

src/builder/builder.rs

@@ -1,8 +1,6 @@
 use crate::builder::types::Action;
-use crate::db::packages;
-use crate::db::prelude::Packages;
 use crate::repo::repo::add_pkg;
-use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait, Set};
+use sea_orm::{ActiveModelTrait, DatabaseConnection, Set};
 use tokio::sync::broadcast::Sender;
 
 pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
@@ -10,24 +8,24 @@ pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
         if let Ok(_result) = tx.subscribe().recv().await {
             match _result {
                 // add a package to parallel build
-                Action::Build(name, version, url, id) => {
+                Action::Build(name, version, url, mut version_model) => {
                     let db = db.clone();
+
+                    // spawn new thread for each pkg build
                     tokio::spawn(async move {
                         match add_pkg(url, version, name).await {
-                            Ok(_) => {
+                            Ok(pkg_file_name) => {
                                 println!("successfully built package");
 
-                                let mut pkg: packages::ActiveModel = Packages::find_by_id(id)
+                                // update status
-                                    .one(&db)
+                                version_model.status = Set(Some(1));
-                                    .await
+                                version_model.file_name = Set(Some(pkg_file_name));
-                                    .unwrap()
+                                version_model.update(&db).await.unwrap();
-                                    .unwrap()
-                                    .into();
-
-                                pkg.status = Set(2);
-                                let pkg: packages::Model = pkg.update(&db).await.unwrap();
                             }
                             Err(e) => {
+                                version_model.status = Set(Some(2));
+                                version_model.update(&db).await.unwrap();
+
                                 println!("Error: {e}")
                             }
                         }

src/builder/types.rs

@@ -1,4 +1,6 @@
+use crate::db::versions;
+
 #[derive(Clone)]
 pub enum Action {
-    Build(String, String, String, i32),
+    Build(String, String, String, versions::ActiveModel),
 }

src/db/builds.rs

@@ -0,0 +1,22 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
+
+use rocket::serde::Serialize;
+use rocket_okapi::okapi::schemars;
+use rocket_okapi::JsonSchema;
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
+#[sea_orm(table_name = "builds")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: i32,
+    pub pkg_id: i32,
+    pub version_id: i32,
+    pub ouput: Option<String>,
+    pub status: Option<i32>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/db/migration/create.rs

@@ -0,0 +1,59 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        let db = manager.get_connection();
+
+        // Use `execute_unprepared` if the SQL statement doesn't have value bindings
+        db.execute_unprepared(
+            r#"
+create table builds
+(
+	id integer not null
+		constraint builds_pk
+			primary key autoincrement,
+	pkg_id integer not null,
+	version_id integer not null,
+	ouput TEXT,
+	status integer
+);
+
+create table packages
+(
+	id integer not null
+		primary key autoincrement,
+	name text not null
+);
+
+create table status
+(
+	id integer not null
+		constraint status_pk
+			primary key autoincrement,
+	value TEXT
+);
+
+create table versions
+(
+	id integer not null
+		constraint versions_pk
+			primary key autoincrement,
+	version TEXT not null,
+	package_id integer not null,
+	file_name TEXT,
+	status INTEGER
+);
+            "#,
+        )
+        .await?;
+        Ok(())
+    }
+
+    async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> {
+        Ok(())
+    }
+}

src/db/migration/m20220101_000001_create_table.rs

@@ -1,49 +0,0 @@
-use sea_orm_migration::prelude::*;
-
-#[derive(DeriveMigrationName)]
-pub struct Migration;
-
-#[async_trait::async_trait]
-impl MigrationTrait for Migration {
-    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
-        manager
-            .create_table(
-                Table::create()
-                    .table(Packages::Table)
-                    .if_not_exists()
-                    .col(
-                        ColumnDef::new(Packages::Id)
-                            .integer()
-                            .not_null()
-                            .auto_increment()
-                            .primary_key(),
-                    )
-                    .col(ColumnDef::new(Packages::Version).string().not_null())
-                    .col(ColumnDef::new(Packages::name).string().not_null())
-                    .col(
-                        ColumnDef::new(Packages::Status)
-                            .integer()
-                            .not_null()
-                            .default(0),
-                    )
-                    .to_owned(),
-            )
-            .await
-    }
-
-    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
-        manager
-            .drop_table(Table::drop().table(Packages::Table).to_owned())
-            .await
-    }
-}
-
-/// Learn more at https://docs.rs/sea-query#iden
-#[derive(Iden)]
-enum Packages {
-    Table,
-    name,
-    Version,
-    Id,
-    Status,
-}

src/db/migration/mod.rs

@@ -1,12 +1,12 @@
 pub use sea_orm_migration::prelude::*;
 
-mod m20220101_000001_create_table;
+mod create;
 
 pub struct Migrator;
 
 #[async_trait::async_trait]
 impl MigratorTrait for Migrator {
     fn migrations() -> Vec<Box<dyn MigrationTrait>> {
-        vec![Box::new(m20220101_000001_create_table::Migration)]
+        vec![Box::new(create::Migration)]
     }
 }

src/db/mod.rs

@@ -2,5 +2,8 @@
 
 pub mod prelude;
 
+pub mod builds;
 pub mod migration;
 pub mod packages;
+pub mod status;
+pub mod versions;

src/db/packages.rs

@@ -8,14 +8,21 @@ use sea_orm::entity::prelude::*;
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
 #[sea_orm(table_name = "packages")]
 pub struct Model {
-    pub name: String,
+    #[sea_orm(primary_key)]
-    pub version: String,
-    #[sea_orm(primary_key, auto_increment = false)]
     pub id: i32,
-    pub status: i32,
+    pub name: String,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {}
+pub enum Relation {
+    #[sea_orm(has_many = "super::versions::Entity")]
+    Versions,
+}
 
 impl ActiveModelBehavior for ActiveModel {}
+
+impl Related<super::versions::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Versions.def()
+    }
+}

src/db/prelude.rs

@@ -1,3 +1,6 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
 
+pub use super::builds::Entity as Builds;
 pub use super::packages::Entity as Packages;
+pub use super::status::Entity as Status;
+pub use super::versions::Entity as Versions;

src/db/status.rs

@@ -0,0 +1,19 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
+
+use rocket::serde::Serialize;
+use rocket_okapi::okapi::schemars;
+use rocket_okapi::JsonSchema;
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
+#[sea_orm(table_name = "status")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: i32,
+    pub value: Option<String>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

src/db/versions.rs

@@ -0,0 +1,36 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
+
+use rocket::serde::Serialize;
+use rocket_okapi::okapi::schemars;
+use rocket_okapi::JsonSchema;
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
+#[sea_orm(table_name = "versions")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: i32,
+    pub version: String,
+    pub package_id: i32,
+    pub file_name: Option<String>,
+    pub status: Option<i32>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+    #[sea_orm(
+        belongs_to = "super::packages::Entity",
+        from = "Column::PackageId",
+        to = "super::packages::Column::Id"
+    )]
+    Packages,
+}
+
+// `Related` trait has to be implemented by hand
+impl Related<super::packages::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Packages.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

src/main.rs

@@ -5,11 +5,12 @@ mod db;
 mod pkgbuild;
 mod repo;
 
-use crate::api::{backend, repository};
+use crate::api::backend;
 use crate::aur::aur::query_aur;
 use crate::builder::types::Action;
 use crate::db::migration::Migrator;
 use rocket::config::Config;
+use rocket::fs::FileServer;
 use rocket::futures::future::join_all;
 use rocket_okapi::swagger_ui::{make_swagger_ui, SwaggerUIConfig};
 use sea_orm::{Database, DatabaseConnection};
@@ -23,17 +24,19 @@ fn main() {
     let (tx, _) = broadcast::channel::<Action>(32);
 
     t.block_on(async move {
-        //build_package("sea-orm-cli").await;
+        // create folder for db stuff
+        if !fs::metadata("./db").is_ok() {
+            fs::create_dir("./db").unwrap();
+        }
 
-        let db: DatabaseConnection = Database::connect("sqlite://db.sqlite?mode=rwc")
+        let db: DatabaseConnection = Database::connect("sqlite://db/db.sqlite?mode=rwc")
             .await
             .unwrap();
 
         Migrator::up(&db, None).await.unwrap();
 
-        // Check if the directory exists
+        // create repo folder
         if !fs::metadata("./repo").is_ok() {
-            // Create the directory if it does not exist
             fs::create_dir("./repo").unwrap();
         }
 
@@ -73,7 +76,7 @@ fn main() {
             config.port = 8080;
 
             let launch_result = rocket::custom(config)
-                .mount("/", repository::build_api())
+                .mount("/", FileServer::from("./repo"))
                 .launch()
                 .await;
             match launch_result {

src/pkgbuild/build.rs

@@ -47,7 +47,7 @@ pub fn build_pkgbuild(
     }
 
     // check if expected built dir exists
-    let built_name = build_repo_packagename(pkg_name.to_string(), pkg_vers.to_string());
+    let built_name = build_expected_repo_packagename(pkg_name.to_string(), pkg_vers.to_string());
     if fs::metadata(format!("{folder_path}/{built_name}")).is_ok() {
         println!("Built {built_name}");
         return Ok(built_name.to_string());
@@ -90,6 +90,8 @@ pub fn build_pkgbuild(
     Err(anyhow!("No package built"))
 }
 
-pub fn build_repo_packagename(pkg_name: String, pkg_vers: String) -> String {
+/// don't trust this pkg name from existing
+/// pkgbuild might build different version name
+pub fn build_expected_repo_packagename(pkg_name: String, pkg_vers: String) -> String {
     format!("{pkg_name}-{pkg_vers}-x86_64.pkg.tar.zst")
 }

src/repo/repo.rs

@@ -1,13 +1,17 @@
 use crate::aur::aur::download_pkgbuild;
-use crate::pkgbuild::build::{build_pkgbuild, build_repo_packagename};
+use crate::db::prelude::Packages;
+use crate::db::prelude::Versions;
+use crate::db::{versions};
+use crate::pkgbuild::build::build_pkgbuild;
 use anyhow::anyhow;
+use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter};
 use std::fs;
 use std::process::Command;
 
 static REPO_NAME: &str = "repo";
 static BASEURL: &str = "https://aur.archlinux.org";
 
-pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<()> {
+pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<String> {
     let fname = download_pkgbuild(format!("{}{}", BASEURL, url).as_str(), "./builds").await?;
     let pkg_file_name =
         build_pkgbuild(format!("./builds/{fname}"), version.as_str(), name.as_str())?;
@@ -19,16 +23,16 @@ pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Resu
     )?;
     fs::remove_file(format!("./builds/{fname}/{pkg_file_name}"))?;
 
-    repo_add(pkg_file_name)?;
+    repo_add(pkg_file_name.clone())?;
 
-    Ok(())
+    Ok(pkg_file_name)
 }
 
 fn repo_add(pkg_file_name: String) -> anyhow::Result<()> {
     let db_file = format!("{REPO_NAME}.db.tar.gz");
 
     let output = Command::new("repo-add")
-        .args(&[db_file.clone(), pkg_file_name])
+        .args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()])
         .current_dir("./repo/")
         .output()?;
 
@@ -48,7 +52,7 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> {
     let db_file = format!("{REPO_NAME}.db.tar.gz");
 
     let output = Command::new("repo-remove")
-        .args(&[db_file.clone(), pkg_file_name])
+        .args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()])
         .current_dir("./repo/")
         .output()?;
 
@@ -64,13 +68,51 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> {
     Ok(())
 }
 
-pub async fn remove_pkg(pkg_name: String, pkg_version: String) -> anyhow::Result<()> {
+pub async fn remove_pkg(db: &DatabaseConnection, pkg_id: i32) -> anyhow::Result<()> {
-    fs::remove_dir_all(format!("./builds/{pkg_name}"))?;
+    let pkg = Packages::find_by_id(pkg_id).one(db).await?.ok_or(anyhow!("id not found"))?;
 
-    let filename = build_repo_packagename(pkg_name.clone(), pkg_version);
+    fs::remove_dir_all(format!("./builds/{}", pkg.name))?;
-    fs::remove_file(format!("./repo/{filename}"))?;
 
-    repo_remove(pkg_name)?;
+    let versions = Versions::find()
+        .filter(versions::Column::PackageId.eq(pkg.id))
+        .all(db)
+        .await?;
+
+    for v in versions {
+        rem_ver(db, v).await?;
+    }
+
+    // remove package db entry
+    pkg.delete(db).await?;
 
     Ok(())
 }
+
+pub async fn remove_version(db: &DatabaseConnection, version_id: i32) -> anyhow::Result<()> {
+    let version = Versions::find()
+        .filter(versions::Column::PackageId.eq(version_id))
+        .one(db)
+        .await?;
+    if let Some(version) = version {
+        rem_ver(db, version).await?;
+    }
+    Ok(())
+}
+
+async fn rem_ver(db: &DatabaseConnection, version: versions::Model) -> anyhow::Result<()> {
+    if let Some(filename) = version.file_name.clone() {
+        // so repo-remove only supports passing a package name and removing the whole package
+        // it seems that repo-add removes an older version when called
+        // todo fix in future by implementing in rust
+        if let Some(pkg) = Packages::find_by_id(version.package_id).one(db).await? {
+            // remove from repo db
+            repo_remove(pkg.name)?;
+
+            // remove from fs
+            fs::remove_file(format!("./repo/{filename}"))?;
+        }
+    }
+
+    version.delete(db).await?;
+    Ok(())
+}