lukas/aurcache
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add improved db layout with pkg versions

Browse Source 
better error handling of api
This commit is contained in:
lukas-heiligenbrunner 2023-12-23 23:00:30 +01:00
parent 9c23bf2411
commit 5bcd8d2ee2
21 changed files with 300 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -3,10 +3,6 @@
debug/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk

1
Dockerfile
View File

@ -30,4 +30,5 @@ RUN pacman -Sc
# EXPOSE 8080
# Set the entry point or default command to run your application
WORKDIR /app
CMD ["untitled"]

15
README.md
View File

@ -1,2 +1,17 @@
# AURCache
A cache build server for Archlinux AUR packages and serving them
## Things still missing
* proper error return to api
* package updates
* multiple package versions
* error checks if requested package does not exist
* proper logging
* auto update packages
* built package version differs from aur pkg version eg. mesa-git
* implement repo-add in rust
* cicd
* build table where all version builds are with stdout
* endpoint to get build log

2
scripts/README.md Normal file
View File

@ -0,0 +1,2 @@
This is a patched makepkg version to allow being run as root.
Especially in containers this makes things a lot easier and shoudln't be a security concern there.

66
src/api/backend.rs
View File

@ -1,6 +1,6 @@
use crate::aur::aur::get_info_by_name;
use crate::builder::types::Action;
use crate::db::packages;
use crate::db::{packages, versions};
use crate::query_aur;
use rocket::serde::json::Json;
use rocket::serde::{Deserialize, Serialize};
@ -8,12 +8,13 @@ use rocket::State;
use rocket::{get, post, Route};
use rocket_okapi::okapi::schemars;
use rocket_okapi::{openapi, openapi_get_routes, JsonSchema};
use sea_orm::{ActiveModelTrait, DatabaseConnection, Set};
use sea_orm::{DeleteResult, EntityTrait, ModelTrait};
use sea_orm::EntityTrait;
use sea_orm::{ActiveModelTrait, DatabaseConnection, FromQueryResult, JoinType, QuerySelect, Set};
use sea_orm::{ColumnTrait, RelationTrait};
use tokio::sync::broadcast::Sender;
use crate::db::prelude::Packages;
use crate::repo::repo::remove_pkg;
use crate::repo::repo::{remove_pkg, remove_version};
#[derive(Serialize, JsonSchema)]
#[serde(crate = "rocket::serde")]
@ -42,14 +43,30 @@ async fn search(query: &str) -> Result<Json<Vec<ApiPackage>>, String> {
}
}
#[derive(FromQueryResult, Deserialize, JsonSchema, Serialize)]
#[serde(crate = "rocket::serde")]
struct ListPackageModel {
name: String,
count: i32,
}
#[openapi(tag = "test")]
#[get("/packages/list")]
async fn package_list(
db: &State<DatabaseConnection>,
) -> Result<Json<Vec<packages::Model>>, String> {
) -> Result<Json<Vec<ListPackageModel>>, String> {
let db = db as &DatabaseConnection;
let all: Vec<packages::Model> = Packages::find().all(db).await.unwrap();
let all: Vec<ListPackageModel> = Packages::find()
.join_rev(JoinType::InnerJoin, versions::Relation::Packages.def())
.select_only()
.column_as(versions::Column::Id.count(), "count")
.column(packages::Column::Name)
.group_by(packages::Column::Name)
.into_model::<ListPackageModel>()
.all(db)
.await
.unwrap();
Ok(Json(all))
}
@ -68,7 +85,6 @@ async fn package_add(
tx: &State<Sender<Action>>,
) -> Result<(), String> {
let db = db as &DatabaseConnection;
let pkg_name = &input.name;
let pkg = get_info_by_name(pkg_name)
@ -77,17 +93,24 @@ async fn package_add(
let new_package = packages::ActiveModel {
name: Set(pkg_name.clone()),
version: Set(pkg.version.clone()),
..Default::default()
};
let t = new_package.save(db).await.expect("TODO: panic message");
let pkt_model = new_package.save(db).await.expect("TODO: panic message");
let new_version = versions::ActiveModel {
version: Set(pkg.version.clone()),
package_id: Set(pkt_model.id.clone().unwrap()),
..Default::default()
};
let version_model = new_version.save(db).await.expect("TODO: panic message");
let _ = tx.send(Action::Build(
pkg.name,
pkg.version,
pkg.url_path.unwrap(),
t.id.unwrap(),
version_model,
));
Ok(())
@ -103,22 +126,23 @@ struct DelBody {
#[post("/packages/delete", data = "<input>")]
async fn package_del(db: &State<DatabaseConnection>, input: Json<DelBody>) -> Result<(), String> {
let db = db as &DatabaseConnection;
let pkg_id = &input.id;
let pkg_id = input.id.clone();
let pkg = Packages::find_by_id(*pkg_id)
.one(db)
.await
.unwrap()
.unwrap();
remove_pkg(db, pkg_id).await.map_err(|e| e.to_string())?;
// remove folders
remove_pkg(pkg.name.to_string(), pkg.version.to_string()).await;
Ok(())
}
#[openapi(tag = "test")]
#[post("/versions/delete/<id>")]
async fn version_del(db: &State<DatabaseConnection>, id: i32) -> Result<(), String> {
let db = db as &DatabaseConnection;
remove_version(db, id).await.map_err(|e| e.to_string())?;
// remove package db entry
let res: DeleteResult = pkg.delete(db).await.unwrap();
Ok(())
}
pub fn build_api() -> Vec<Route> {
openapi_get_routes![search, package_list, package_add, package_del]
openapi_get_routes![search, package_list, package_add, package_del, version_del]
}

1
src/api/mod.rs
View File

@ -1,2 +1 @@
pub mod backend;
pub mod repository;

5
src/api/repository.rs
View File

@ -1,5 +0,0 @@
use rocket::fs::FileServer;
pub fn build_api() -> FileServer {
FileServer::from("./repo")
}

26
src/builder/builder.rs
View File

@ -1,8 +1,6 @@
use crate::builder::types::Action;
use crate::db::packages;
use crate::db::prelude::Packages;
use crate::repo::repo::add_pkg;
use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait, Set};
use sea_orm::{ActiveModelTrait, DatabaseConnection, Set};
use tokio::sync::broadcast::Sender;
pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
@ -10,24 +8,24 @@ pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
if let Ok(_result) = tx.subscribe().recv().await {
match _result {
// add a package to parallel build
Action::Build(name, version, url, id) => {
Action::Build(name, version, url, mut version_model) => {
let db = db.clone();
// spawn new thread for each pkg build
tokio::spawn(async move {
match add_pkg(url, version, name).await {
Ok(_) => {
Ok(pkg_file_name) => {
println!("successfully built package");
let mut pkg: packages::ActiveModel = Packages::find_by_id(id)
.one(&db)
.await
.unwrap()
.unwrap()
.into();
pkg.status = Set(2);
let pkg: packages::Model = pkg.update(&db).await.unwrap();
// update status
version_model.status = Set(Some(1));
version_model.file_name = Set(Some(pkg_file_name));
version_model.update(&db).await.unwrap();
}
Err(e) => {
version_model.status = Set(Some(2));
version_model.update(&db).await.unwrap();
println!("Error: {e}")
}
}

4
src/builder/types.rs
View File

@ -1,4 +1,6 @@
use crate::db::versions;
#[derive(Clone)]
pub enum Action {
Build(String, String, String, i32),
Build(String, String, String, versions::ActiveModel),
}

22
src/db/builds.rs Normal file
View File

@ -0,0 +1,22 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
use rocket::serde::Serialize;
use rocket_okapi::okapi::schemars;
use rocket_okapi::JsonSchema;
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
#[sea_orm(table_name = "builds")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i32,
pub pkg_id: i32,
pub version_id: i32,
pub ouput: Option<String>,
pub status: Option<i32>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}

59
src/db/migration/create.rs Normal file
View File

@ -0,0 +1,59 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
let db = manager.get_connection();
// Use `execute_unprepared` if the SQL statement doesn't have value bindings
db.execute_unprepared(
r#"
create table builds
(
id integer not null
constraint builds_pk
primary key autoincrement,
pkg_id integer not null,
version_id integer not null,
ouput TEXT,
status integer
);
create table packages
(
id integer not null
primary key autoincrement,
name text not null
);
create table status
(
id integer not null
constraint status_pk
primary key autoincrement,
value TEXT
);
create table versions
(
id integer not null
constraint versions_pk
primary key autoincrement,
version TEXT not null,
package_id integer not null,
file_name TEXT,
status INTEGER
);
"#,
)
.await?;
Ok(())
}
async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> {
Ok(())
}
}

49
src/db/migration/m20220101_000001_create_table.rs
View File

@ -1,49 +0,0 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_table(
Table::create()
.table(Packages::Table)
.if_not_exists()
.col(
ColumnDef::new(Packages::Id)
.integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(Packages::Version).string().not_null())
.col(ColumnDef::new(Packages::name).string().not_null())
.col(
ColumnDef::new(Packages::Status)
.integer()
.not_null()
.default(0),
)
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_table(Table::drop().table(Packages::Table).to_owned())
.await
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum Packages {
Table,
name,
Version,
Id,
Status,
}

4
src/db/migration/mod.rs
View File

@ -1,12 +1,12 @@
pub use sea_orm_migration::prelude::*;
mod m20220101_000001_create_table;
mod create;
pub struct Migrator;
#[async_trait::async_trait]
impl MigratorTrait for Migrator {
fn migrations() -> Vec<Box<dyn MigrationTrait>> {
vec![Box::new(m20220101_000001_create_table::Migration)]
vec![Box::new(create::Migration)]
}
}

3
src/db/mod.rs
View File

@ -2,5 +2,8 @@
pub mod prelude;
pub mod builds;
pub mod migration;
pub mod packages;
pub mod status;
pub mod versions;

17
src/db/packages.rs
View File

@ -8,14 +8,21 @@ use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
#[sea_orm(table_name = "packages")]
pub struct Model {
pub name: String,
pub version: String,
#[sea_orm(primary_key, auto_increment = false)]
#[sea_orm(primary_key)]
pub id: i32,
pub status: i32,
pub name: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
pub enum Relation {
#[sea_orm(has_many = "super::versions::Entity")]
Versions,
}
impl ActiveModelBehavior for ActiveModel {}
impl Related<super::versions::Entity> for Entity {
fn to() -> RelationDef {
Relation::Versions.def()
}
}

3
src/db/prelude.rs
View File

@ -1,3 +1,6 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
pub use super::builds::Entity as Builds;
pub use super::packages::Entity as Packages;
pub use super::status::Entity as Status;
pub use super::versions::Entity as Versions;

19
src/db/status.rs Normal file
View File

@ -0,0 +1,19 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
use rocket::serde::Serialize;
use rocket_okapi::okapi::schemars;
use rocket_okapi::JsonSchema;
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
#[sea_orm(table_name = "status")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i32,
pub value: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}

36
src/db/versions.rs Normal file
View File

@ -0,0 +1,36 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.2
use rocket::serde::Serialize;
use rocket_okapi::okapi::schemars;
use rocket_okapi::JsonSchema;
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, JsonSchema)]
#[sea_orm(table_name = "versions")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i32,
pub version: String,
pub package_id: i32,
pub file_name: Option<String>,
pub status: Option<i32>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::packages::Entity",
from = "Column::PackageId",
to = "super::packages::Column::Id"
)]
Packages,
}
// `Related` trait has to be implemented by hand
impl Related<super::packages::Entity> for Entity {
fn to() -> RelationDef {
Relation::Packages.def()
}
}
impl ActiveModelBehavior for ActiveModel {}

15
src/main.rs
View File

@ -5,11 +5,12 @@ mod db;
mod pkgbuild;
mod repo;
use crate::api::{backend, repository};
use crate::api::backend;
use crate::aur::aur::query_aur;
use crate::builder::types::Action;
use crate::db::migration::Migrator;
use rocket::config::Config;
use rocket::fs::FileServer;
use rocket::futures::future::join_all;
use rocket_okapi::swagger_ui::{make_swagger_ui, SwaggerUIConfig};
use sea_orm::{Database, DatabaseConnection};
@ -23,17 +24,19 @@ fn main() {
let (tx, _) = broadcast::channel::<Action>(32);
t.block_on(async move {
//build_package("sea-orm-cli").await;
// create folder for db stuff
if !fs::metadata("./db").is_ok() {
fs::create_dir("./db").unwrap();
}
let db: DatabaseConnection = Database::connect("sqlite://db.sqlite?mode=rwc")
let db: DatabaseConnection = Database::connect("sqlite://db/db.sqlite?mode=rwc")
.await
.unwrap();
Migrator::up(&db, None).await.unwrap();
// Check if the directory exists
// create repo folder
if !fs::metadata("./repo").is_ok() {
// Create the directory if it does not exist
fs::create_dir("./repo").unwrap();
}
@ -73,7 +76,7 @@ fn main() {
config.port = 8080;
let launch_result = rocket::custom(config)
.mount("/", repository::build_api())
.mount("/", FileServer::from("./repo"))
.launch()
.await;
match launch_result {

6
src/pkgbuild/build.rs
View File

@ -47,7 +47,7 @@ pub fn build_pkgbuild(
}
// check if expected built dir exists
let built_name = build_repo_packagename(pkg_name.to_string(), pkg_vers.to_string());
let built_name = build_expected_repo_packagename(pkg_name.to_string(), pkg_vers.to_string());
if fs::metadata(format!("{folder_path}/{built_name}")).is_ok() {
println!("Built {built_name}");
return Ok(built_name.to_string());
@ -90,6 +90,8 @@ pub fn build_pkgbuild(
Err(anyhow!("No package built"))
}
pub fn build_repo_packagename(pkg_name: String, pkg_vers: String) -> String {
/// don't trust this pkg name from existing
/// pkgbuild might build different version name
pub fn build_expected_repo_packagename(pkg_name: String, pkg_vers: String) -> String {
format!("{pkg_name}-{pkg_vers}-x86_64.pkg.tar.zst")
}

64
src/repo/repo.rs
View File

@ -1,13 +1,17 @@
use crate::aur::aur::download_pkgbuild;
use crate::pkgbuild::build::{build_pkgbuild, build_repo_packagename};
use crate::db::prelude::Packages;
use crate::db::prelude::Versions;
use crate::db::{versions};
use crate::pkgbuild::build::build_pkgbuild;
use anyhow::anyhow;
use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter};
use std::fs;
use std::process::Command;
static REPO_NAME: &str = "repo";
static BASEURL: &str = "https://aur.archlinux.org";
pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<()> {
pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<String> {
let fname = download_pkgbuild(format!("{}{}", BASEURL, url).as_str(), "./builds").await?;
let pkg_file_name =
build_pkgbuild(format!("./builds/{fname}"), version.as_str(), name.as_str())?;
@ -19,16 +23,16 @@ pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Resu
)?;
fs::remove_file(format!("./builds/{fname}/{pkg_file_name}"))?;
repo_add(pkg_file_name)?;
repo_add(pkg_file_name.clone())?;
Ok(())
Ok(pkg_file_name)
}
fn repo_add(pkg_file_name: String) -> anyhow::Result<()> {
let db_file = format!("{REPO_NAME}.db.tar.gz");
let output = Command::new("repo-add")
.args(&[db_file.clone(), pkg_file_name])
.args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()])
.current_dir("./repo/")
.output()?;
@ -48,7 +52,7 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> {
let db_file = format!("{REPO_NAME}.db.tar.gz");
let output = Command::new("repo-remove")
.args(&[db_file.clone(), pkg_file_name])
.args(&[db_file.clone(), pkg_file_name, "--nocolor".to_string()])
.current_dir("./repo/")
.output()?;
@ -64,13 +68,51 @@ fn repo_remove(pkg_file_name: String) -> anyhow::Result<()> {
Ok(())
}
pub async fn remove_pkg(pkg_name: String, pkg_version: String) -> anyhow::Result<()> {
fs::remove_dir_all(format!("./builds/{pkg_name}"))?;
pub async fn remove_pkg(db: &DatabaseConnection, pkg_id: i32) -> anyhow::Result<()> {
let pkg = Packages::find_by_id(pkg_id).one(db).await?.ok_or(anyhow!("id not found"))?;
let filename = build_repo_packagename(pkg_name.clone(), pkg_version);
fs::remove_file(format!("./repo/{filename}"))?;
fs::remove_dir_all(format!("./builds/{}", pkg.name))?;
repo_remove(pkg_name)?;
let versions = Versions::find()
.filter(versions::Column::PackageId.eq(pkg.id))
.all(db)
.await?;
for v in versions {
rem_ver(db, v).await?;
}
// remove package db entry
pkg.delete(db).await?;
Ok(())
}
pub async fn remove_version(db: &DatabaseConnection, version_id: i32) -> anyhow::Result<()> {
let version = Versions::find()
.filter(versions::Column::PackageId.eq(version_id))
.one(db)
.await?;
if let Some(version) = version {
rem_ver(db, version).await?;
}
Ok(())
}
async fn rem_ver(db: &DatabaseConnection, version: versions::Model) -> anyhow::Result<()> {
if let Some(filename) = version.file_name.clone() {
// so repo-remove only supports passing a package name and removing the whole package
// it seems that repo-add removes an older version when called
// todo fix in future by implementing in rust
if let Some(pkg) = Packages::find_by_id(version.package_id).one(db).await? {
// remove from repo db
repo_remove(pkg.name)?;
// remove from fs
fs::remove_file(format!("./repo/{filename}"))?;
}
}
version.delete(db).await?;
Ok(())
}