From 5bcd8d2ee2dae1055a94db1c233d406d5b48cb00 Mon Sep 17 00:00:00 2001 From: lukas-heiligenbrunner Date: Sat, 23 Dec 2023 23:00:30 +0100 Subject: [PATCH] add improved db layout with pkg versions better error handling of api --- .gitignore | 4 -- Dockerfile | 1 + README.md | 15 +++++ scripts/README.md | 2 + src/api/backend.rs | 66 +++++++++++++------ src/api/mod.rs | 1 - src/api/repository.rs | 5 -- src/builder/builder.rs | 26 ++++---- src/builder/types.rs | 4 +- src/db/builds.rs | 22 +++++++ src/db/migration/create.rs | 59 +++++++++++++++++ .../m20220101_000001_create_table.rs | 49 -------------- src/db/migration/mod.rs | 4 +- src/db/mod.rs | 3 + src/db/packages.rs | 17 +++-- src/db/prelude.rs | 3 + src/db/status.rs | 19 ++++++ src/db/versions.rs | 36 ++++++++++ src/main.rs | 15 +++-- src/pkgbuild/build.rs | 6 +- src/repo/repo.rs | 64 ++++++++++++++---- 21 files changed, 300 insertions(+), 121 deletions(-) create mode 100644 scripts/README.md delete mode 100644 src/api/repository.rs create mode 100644 src/db/builds.rs create mode 100644 src/db/migration/create.rs delete mode 100644 src/db/migration/m20220101_000001_create_table.rs create mode 100644 src/db/status.rs create mode 100644 src/db/versions.rs diff --git a/.gitignore b/.gitignore index 6985cf1..73fab07 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,6 @@ debug/ target/ -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Dockerfile b/Dockerfile index 735da29..ef63db1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,4 +30,5 @@ RUN pacman -Sc # EXPOSE 8080 # Set the entry point or default command to run your application +WORKDIR /app CMD ["untitled"] diff --git a/README.md b/README.md index 77e1339..002c3b2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ # AURCache A cache build server for Archlinux AUR packages and serving them + + +## Things still missing + +* proper error return to api +* package updates +* multiple package versions +* error checks if requested package does not exist +* proper logging +* auto update packages +* built package version differs from aur pkg version eg. mesa-git +* implement repo-add in rust +* cicd +* build table where all version builds are with stdout +* endpoint to get build log \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..c32607e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,2 @@ +This is a patched makepkg version to allow being run as root. +Especially in containers this makes things a lot easier and shoudln't be a security concern there. diff --git a/src/api/backend.rs b/src/api/backend.rs index bed600c..fcfa75a 100644 --- a/src/api/backend.rs +++ b/src/api/backend.rs @@ -1,6 +1,6 @@ use crate::aur::aur::get_info_by_name; use crate::builder::types::Action; -use crate::db::packages; +use crate::db::{packages, versions}; use crate::query_aur; use rocket::serde::json::Json; use rocket::serde::{Deserialize, Serialize}; @@ -8,12 +8,13 @@ use rocket::State; use rocket::{get, post, Route}; use rocket_okapi::okapi::schemars; use rocket_okapi::{openapi, openapi_get_routes, JsonSchema}; -use sea_orm::{ActiveModelTrait, DatabaseConnection, Set}; -use sea_orm::{DeleteResult, EntityTrait, ModelTrait}; +use sea_orm::EntityTrait; +use sea_orm::{ActiveModelTrait, DatabaseConnection, FromQueryResult, JoinType, QuerySelect, Set}; +use sea_orm::{ColumnTrait, RelationTrait}; use tokio::sync::broadcast::Sender; use crate::db::prelude::Packages; -use crate::repo::repo::remove_pkg; +use crate::repo::repo::{remove_pkg, remove_version}; #[derive(Serialize, JsonSchema)] #[serde(crate = "rocket::serde")] @@ -42,14 +43,30 @@ async fn search(query: &str) -> Result>, String> { } } +#[derive(FromQueryResult, Deserialize, JsonSchema, Serialize)] +#[serde(crate = "rocket::serde")] +struct ListPackageModel { + name: String, + count: i32, +} + #[openapi(tag = "test")] #[get("/packages/list")] async fn package_list( db: &State, -) -> Result>, String> { +) -> Result>, String> { let db = db as &DatabaseConnection; - let all: Vec = Packages::find().all(db).await.unwrap(); + let all: Vec = Packages::find() + .join_rev(JoinType::InnerJoin, versions::Relation::Packages.def()) + .select_only() + .column_as(versions::Column::Id.count(), "count") + .column(packages::Column::Name) + .group_by(packages::Column::Name) + .into_model::() + .all(db) + .await + .unwrap(); Ok(Json(all)) } @@ -68,7 +85,6 @@ async fn package_add( tx: &State>, ) -> Result<(), String> { let db = db as &DatabaseConnection; - let pkg_name = &input.name; let pkg = get_info_by_name(pkg_name) @@ -77,17 +93,24 @@ async fn package_add( let new_package = packages::ActiveModel { name: Set(pkg_name.clone()), - version: Set(pkg.version.clone()), ..Default::default() }; - let t = new_package.save(db).await.expect("TODO: panic message"); + let pkt_model = new_package.save(db).await.expect("TODO: panic message"); + + let new_version = versions::ActiveModel { + version: Set(pkg.version.clone()), + package_id: Set(pkt_model.id.clone().unwrap()), + ..Default::default() + }; + + let version_model = new_version.save(db).await.expect("TODO: panic message"); let _ = tx.send(Action::Build( pkg.name, pkg.version, pkg.url_path.unwrap(), - t.id.unwrap(), + version_model, )); Ok(()) @@ -103,22 +126,23 @@ struct DelBody { #[post("/packages/delete", data = "")] async fn package_del(db: &State, input: Json) -> Result<(), String> { let db = db as &DatabaseConnection; - let pkg_id = &input.id; + let pkg_id = input.id.clone(); - let pkg = Packages::find_by_id(*pkg_id) - .one(db) - .await - .unwrap() - .unwrap(); + remove_pkg(db, pkg_id).await.map_err(|e| e.to_string())?; - // remove folders - remove_pkg(pkg.name.to_string(), pkg.version.to_string()).await; + Ok(()) +} + +#[openapi(tag = "test")] +#[post("/versions/delete/")] +async fn version_del(db: &State, id: i32) -> Result<(), String> { + let db = db as &DatabaseConnection; + + remove_version(db, id).await.map_err(|e| e.to_string())?; - // remove package db entry - let res: DeleteResult = pkg.delete(db).await.unwrap(); Ok(()) } pub fn build_api() -> Vec { - openapi_get_routes![search, package_list, package_add, package_del] + openapi_get_routes![search, package_list, package_add, package_del, version_del] } diff --git a/src/api/mod.rs b/src/api/mod.rs index f771c2f..fceb141 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,2 +1 @@ pub mod backend; -pub mod repository; diff --git a/src/api/repository.rs b/src/api/repository.rs deleted file mode 100644 index 2e6a364..0000000 --- a/src/api/repository.rs +++ /dev/null @@ -1,5 +0,0 @@ -use rocket::fs::FileServer; - -pub fn build_api() -> FileServer { - FileServer::from("./repo") -} diff --git a/src/builder/builder.rs b/src/builder/builder.rs index c9e70e2..3094b17 100644 --- a/src/builder/builder.rs +++ b/src/builder/builder.rs @@ -1,8 +1,6 @@ use crate::builder::types::Action; -use crate::db::packages; -use crate::db::prelude::Packages; use crate::repo::repo::add_pkg; -use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}; +use sea_orm::{ActiveModelTrait, DatabaseConnection, Set}; use tokio::sync::broadcast::Sender; pub async fn init(db: DatabaseConnection, tx: Sender) { @@ -10,24 +8,24 @@ pub async fn init(db: DatabaseConnection, tx: Sender) { if let Ok(_result) = tx.subscribe().recv().await { match _result { // add a package to parallel build - Action::Build(name, version, url, id) => { + Action::Build(name, version, url, mut version_model) => { let db = db.clone(); + + // spawn new thread for each pkg build tokio::spawn(async move { match add_pkg(url, version, name).await { - Ok(_) => { + Ok(pkg_file_name) => { println!("successfully built package"); - let mut pkg: packages::ActiveModel = Packages::find_by_id(id) - .one(&db) - .await - .unwrap() - .unwrap() - .into(); - - pkg.status = Set(2); - let pkg: packages::Model = pkg.update(&db).await.unwrap(); + // update status + version_model.status = Set(Some(1)); + version_model.file_name = Set(Some(pkg_file_name)); + version_model.update(&db).await.unwrap(); } Err(e) => { + version_model.status = Set(Some(2)); + version_model.update(&db).await.unwrap(); + println!("Error: {e}") } } diff --git a/src/builder/types.rs b/src/builder/types.rs index fd2a5c4..84eea5c 100644 --- a/src/builder/types.rs +++ b/src/builder/types.rs @@ -1,4 +1,6 @@ +use crate::db::versions; + #[derive(Clone)] pub enum Action { - Build(String, String, String, i32), + Build(String, String, String, versions::ActiveModel), } diff --git a/src/db/builds.rs b/src/db/builds.rs new file mode 100644 index 0000000..4375d30 --- /dev/null +++ b/src/db/builds.rs @@ -0,0 +1,22 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2 + +use rocket::serde::Serialize; +use rocket_okapi::okapi::schemars; +use rocket_okapi::JsonSchema; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)] +#[sea_orm(table_name = "builds")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub pkg_id: i32, + pub version_id: i32, + pub ouput: Option, + pub status: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/db/migration/create.rs b/src/db/migration/create.rs new file mode 100644 index 0000000..754a246 --- /dev/null +++ b/src/db/migration/create.rs @@ -0,0 +1,59 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + + // Use `execute_unprepared` if the SQL statement doesn't have value bindings + db.execute_unprepared( + r#" +create table builds +( + id integer not null + constraint builds_pk + primary key autoincrement, + pkg_id integer not null, + version_id integer not null, + ouput TEXT, + status integer +); + +create table packages +( + id integer not null + primary key autoincrement, + name text not null +); + +create table status +( + id integer not null + constraint status_pk + primary key autoincrement, + value TEXT +); + +create table versions +( + id integer not null + constraint versions_pk + primary key autoincrement, + version TEXT not null, + package_id integer not null, + file_name TEXT, + status INTEGER +); + "#, + ) + .await?; + Ok(()) + } + + async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} diff --git a/src/db/migration/m20220101_000001_create_table.rs b/src/db/migration/m20220101_000001_create_table.rs deleted file mode 100644 index 4d1b5b3..0000000 --- a/src/db/migration/m20220101_000001_create_table.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[derive(DeriveMigrationName)] -pub struct Migration; - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(Packages::Table) - .if_not_exists() - .col( - ColumnDef::new(Packages::Id) - .integer() - .not_null() - .auto_increment() - .primary_key(), - ) - .col(ColumnDef::new(Packages::Version).string().not_null()) - .col(ColumnDef::new(Packages::name).string().not_null()) - .col( - ColumnDef::new(Packages::Status) - .integer() - .not_null() - .default(0), - ) - .to_owned(), - ) - .await - } - - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_table(Table::drop().table(Packages::Table).to_owned()) - .await - } -} - -/// Learn more at https://docs.rs/sea-query#iden -#[derive(Iden)] -enum Packages { - Table, - name, - Version, - Id, - Status, -} diff --git a/src/db/migration/mod.rs b/src/db/migration/mod.rs index 2c605af..ac63e89 100644 --- a/src/db/migration/mod.rs +++ b/src/db/migration/mod.rs @@ -1,12 +1,12 @@ pub use sea_orm_migration::prelude::*; -mod m20220101_000001_create_table; +mod create; pub struct Migrator; #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { - vec![Box::new(m20220101_000001_create_table::Migration)] + vec![Box::new(create::Migration)] } } diff --git a/src/db/mod.rs b/src/db/mod.rs index 45f86e6..328c084 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -2,5 +2,8 @@ pub mod prelude; +pub mod builds; pub mod migration; pub mod packages; +pub mod status; +pub mod versions; diff --git a/src/db/packages.rs b/src/db/packages.rs index cea5136..32d1916 100644 --- a/src/db/packages.rs +++ b/src/db/packages.rs @@ -8,14 +8,21 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)] #[sea_orm(table_name = "packages")] pub struct Model { - pub name: String, - pub version: String, - #[sea_orm(primary_key, auto_increment = false)] + #[sea_orm(primary_key)] pub id: i32, - pub status: i32, + pub name: String, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} +pub enum Relation { + #[sea_orm(has_many = "super::versions::Entity")] + Versions, +} impl ActiveModelBehavior for ActiveModel {} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Versions.def() + } +} diff --git a/src/db/prelude.rs b/src/db/prelude.rs index 078bce5..7d08b14 100644 --- a/src/db/prelude.rs +++ b/src/db/prelude.rs @@ -1,3 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2 +pub use super::builds::Entity as Builds; pub use super::packages::Entity as Packages; +pub use super::status::Entity as Status; +pub use super::versions::Entity as Versions; diff --git a/src/db/status.rs b/src/db/status.rs new file mode 100644 index 0000000..a60135e --- /dev/null +++ b/src/db/status.rs @@ -0,0 +1,19 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2 + +use rocket::serde::Serialize; +use rocket_okapi::okapi::schemars; +use rocket_okapi::JsonSchema; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)] +#[sea_orm(table_name = "status")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub value: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/db/versions.rs b/src/db/versions.rs new file mode 100644 index 0000000..66e85b7 --- /dev/null +++ b/src/db/versions.rs @@ -0,0 +1,36 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2 + +use rocket::serde::Serialize; +use rocket_okapi::okapi::schemars; +use rocket_okapi::JsonSchema; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)] +#[sea_orm(table_name = "versions")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub version: String, + pub package_id: i32, + pub file_name: Option, + pub status: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::packages::Entity", + from = "Column::PackageId", + to = "super::packages::Column::Id" + )] + Packages, +} + +// `Related` trait has to be implemented by hand +impl Related for Entity { + fn to() -> RelationDef { + Relation::Packages.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/main.rs b/src/main.rs index dda3237..9d423fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,11 +5,12 @@ mod db; mod pkgbuild; mod repo; -use crate::api::{backend, repository}; +use crate::api::backend; use crate::aur::aur::query_aur; use crate::builder::types::Action; use crate::db::migration::Migrator; use rocket::config::Config; +use rocket::fs::FileServer; use rocket::futures::future::join_all; use rocket_okapi::swagger_ui::{make_swagger_ui, SwaggerUIConfig}; use sea_orm::{Database, DatabaseConnection}; @@ -23,17 +24,19 @@ fn main() { let (tx, _) = broadcast::channel::(32); t.block_on(async move { - //build_package("sea-orm-cli").await; + // create folder for db stuff + if !fs::metadata("./db").is_ok() { + fs::create_dir("./db").unwrap(); + } - let db: DatabaseConnection = Database::connect("sqlite://db.sqlite?mode=rwc") + let db: DatabaseConnection = Database::connect("sqlite://db/db.sqlite?mode=rwc") .await .unwrap(); Migrator::up(&db, None).await.unwrap(); - // Check if the directory exists + // create repo folder if !fs::metadata("./repo").is_ok() { - // Create the directory if it does not exist fs::create_dir("./repo").unwrap(); } @@ -73,7 +76,7 @@ fn main() { config.port = 8080; let launch_result = rocket::custom(config) - .mount("/", repository::build_api()) + .mount("/", FileServer::from("./repo")) .launch() .await; match launch_result { diff --git a/src/pkgbuild/build.rs b/src/pkgbuild/build.rs index 5598c2a..bcf228f 100644 --- a/src/pkgbuild/build.rs +++ b/src/pkgbuild/build.rs @@ -47,7 +47,7 @@ pub fn build_pkgbuild( } // check if expected built dir exists - let built_name = build_repo_packagename(pkg_name.to_string(), pkg_vers.to_string()); + let built_name = build_expected_repo_packagename(pkg_name.to_string(), pkg_vers.to_string()); if fs::metadata(format!("{folder_path}/{built_name}")).is_ok() { println!("Built {built_name}"); return Ok(built_name.to_string()); @@ -90,6 +90,8 @@ pub fn build_pkgbuild( Err(anyhow!("No package built")) } -pub fn build_repo_packagename(pkg_name: String, pkg_vers: String) -> String { +/// don't trust this pkg name from existing +/// pkgbuild might build different version name +pub fn build_expected_repo_packagename(pkg_name: String, pkg_vers: String) -> String { format!("{pkg_name}-{pkg_vers}-x86_64.pkg.tar.zst") } diff --git a/src/repo/repo.rs b/src/repo/repo.rs index c416499..7183676 100644 --- a/src/repo/repo.rs +++ b/src/repo/repo.rs @@ -1,13 +1,17 @@ use crate::aur::aur::download_pkgbuild; -use crate::pkgbuild::build::{build_pkgbuild, build_repo_packagename}; +use crate::db::prelude::Packages; +use crate::db::prelude::Versions; +use crate::db::{versions}; +use crate::pkgbuild::build::build_pkgbuild; use anyhow::anyhow; +use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter}; use std::fs; use std::process::Command; static REPO_NAME: &str = "repo"; static BASEURL: &str = "https://aur.archlinux.org"; -pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<()> { +pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result { let fname = download_pkgbuild(format!("{}{}", BASEURL, url).as_str(), "./builds").await?; let pkg_file_name = build_pkgbuild(format!("./builds/{fname}"), version.as_str(), name.as_str())?; @@ -19,16 +23,16 @@ pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Resu )?; fs::remove_file(format!("./builds/{fname}/{pkg_file_name}"))?; - repo_add(pkg_file_name)?; + repo_add(pkg_file_name.clone())?; - Ok(()) + Ok(pkg_file_name) } fn repo_add(pkg_file_name: String) -> anyhow::Result<()> { let db_file = format!("{REPO_NAME}.db.tar.gz"); let output = Command::new("repo-add") - .args(&[db_file.clone(), pkg_file_name]) + .args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()]) .current_dir("./repo/") .output()?; @@ -48,7 +52,7 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> { let db_file = format!("{REPO_NAME}.db.tar.gz"); let output = Command::new("repo-remove") - .args(&[db_file.clone(), pkg_file_name]) + .args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()]) .current_dir("./repo/") .output()?; @@ -64,13 +68,51 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> { Ok(()) } -pub async fn remove_pkg(pkg_name: String, pkg_version: String) -> anyhow::Result<()> { - fs::remove_dir_all(format!("./builds/{pkg_name}"))?; +pub async fn remove_pkg(db: &DatabaseConnection, pkg_id: i32) -> anyhow::Result<()> { + let pkg = Packages::find_by_id(pkg_id).one(db).await?.ok_or(anyhow!("id not found"))?; - let filename = build_repo_packagename(pkg_name.clone(), pkg_version); - fs::remove_file(format!("./repo/{filename}"))?; + fs::remove_dir_all(format!("./builds/{}", pkg.name))?; - repo_remove(pkg_name)?; + let versions = Versions::find() + .filter(versions::Column::PackageId.eq(pkg.id)) + .all(db) + .await?; + + for v in versions { + rem_ver(db, v).await?; + } + + // remove package db entry + pkg.delete(db).await?; Ok(()) } + +pub async fn remove_version(db: &DatabaseConnection, version_id: i32) -> anyhow::Result<()> { + let version = Versions::find() + .filter(versions::Column::PackageId.eq(version_id)) + .one(db) + .await?; + if let Some(version) = version { + rem_ver(db, version).await?; + } + Ok(()) +} + +async fn rem_ver(db: &DatabaseConnection, version: versions::Model) -> anyhow::Result<()> { + if let Some(filename) = version.file_name.clone() { + // so repo-remove only supports passing a package name and removing the whole package + // it seems that repo-add removes an older version when called + // todo fix in future by implementing in rust + if let Some(pkg) = Packages::find_by_id(version.package_id).one(db).await? { + // remove from repo db + repo_remove(pkg.name)?; + + // remove from fs + fs::remove_file(format!("./repo/{filename}"))?; + } + } + + version.delete(db).await?; + Ok(()) +}