store build logs in db

endpoint to get logs
checks if pkg already in db force build flag
This commit is contained in:
lukas-heiligenbrunner 2023-12-26 21:44:19 +01:00
parent 18f35f948e
commit 406da5325f
9 changed files with 283 additions and 62 deletions

View File

@ -1,12 +1,14 @@
use crate::aur::aur::get_info_by_name;
use crate::builder::types::Action;
use crate::db::prelude::{Packages, Versions};
use crate::db::{packages, versions};
use rocket::response::status::NotFound;
use rocket::serde::json::Json;
use rocket::serde::Deserialize;
use rocket::{post, State};
use rocket_okapi::okapi::schemars;
use rocket_okapi::{openapi, JsonSchema};
use sea_orm::ActiveModelTrait;
use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter};
use sea_orm::{DatabaseConnection, Set};
use tokio::sync::broadcast::Sender;
@ -14,6 +16,7 @@ use tokio::sync::broadcast::Sender;
#[serde(crate = "rocket::serde")]
pub struct AddBody {
name: String,
force_build: bool,
}
#[openapi(tag = "test")]
@ -22,28 +25,55 @@ pub async fn package_add(
db: &State<DatabaseConnection>,
input: Json<AddBody>,
tx: &State<Sender<Action>>,
) -> Result<(), String> {
) -> Result<(), NotFound<String>> {
let db = db as &DatabaseConnection;
let pkg_name = &input.name;
let pkg = get_info_by_name(pkg_name)
let pkt_model = match Packages::find()
.filter(packages::Column::Name.eq(input.name.clone()))
.one(db)
.await
.map_err(|_| "couldn't download package metadata".to_string())?;
.map_err(|e| NotFound(e.to_string()))?
{
None => {
let new_package = packages::ActiveModel {
name: Set(pkg_name.clone()),
name: Set(input.name.clone()),
..Default::default()
};
let pkt_model = new_package.save(db).await.expect("TODO: panic message");
new_package.save(db).await.expect("TODO: panic message")
}
Some(p) => p.into(),
};
let pkg = get_info_by_name(input.name.clone().as_str())
.await
.map_err(|_| NotFound("couldn't download package metadata".to_string()))?;
let version_model = match Versions::find()
.filter(versions::Column::Version.eq(pkg.version.clone()))
.one(db)
.await
.map_err(|e| NotFound(e.to_string()))?
{
None => {
let new_version = versions::ActiveModel {
version: Set(pkg.version.clone()),
package_id: Set(pkt_model.id.clone().unwrap()),
..Default::default()
};
let version_model = new_version.save(db).await.expect("TODO: panic message");
new_version.save(db).await.expect("TODO: panic message")
}
Some(p) => {
// todo add check if this version was successfully built
// if not allow build
if input.force_build {
p.into()
} else {
return Err(NotFound("Version already existing".to_string()));
}
}
};
let _ = tx.send(Action::Build(
pkg.name,

View File

@ -1,7 +1,9 @@
use crate::api::add::okapi_add_operation_for_package_add_;
use crate::api::add::package_add;
use crate::api::list::okapi_add_operation_for_package_list_;
use crate::api::list::okapi_add_operation_for_search_;
use crate::api::list::okapi_add_operation_for_build_output_;
use crate::api::list::okapi_add_operation_for_list_builds_;
use crate::api::list::{build_output, okapi_add_operation_for_package_list_};
use crate::api::list::{list_builds, okapi_add_operation_for_search_};
use crate::api::list::{package_list, search};
use crate::api::remove::okapi_add_operation_for_package_del_;
use crate::api::remove::okapi_add_operation_for_version_del_;
@ -10,5 +12,13 @@ use rocket::Route;
use rocket_okapi::openapi_get_routes;
pub fn build_api() -> Vec<Route> {
openapi_get_routes![search, package_list, package_add, package_del, version_del]
openapi_get_routes![
search,
package_list,
package_add,
package_del,
version_del,
build_output,
list_builds
]
}

View File

@ -1,13 +1,14 @@
use crate::aur::aur::query_aur;
use crate::db::migration::JoinType;
use crate::db::prelude::Packages;
use crate::db::{packages, versions};
use crate::db::prelude::{Builds, Packages};
use crate::db::{builds, packages, versions};
use rocket::response::status::NotFound;
use rocket::serde::json::Json;
use rocket::serde::{Deserialize, Serialize};
use rocket::{get, State};
use rocket_okapi::okapi::schemars;
use rocket_okapi::{openapi, JsonSchema};
use sea_orm::ColumnTrait;
use sea_orm::{ColumnTrait, QueryFilter};
use sea_orm::{DatabaseConnection, EntityTrait, FromQueryResult, QuerySelect, RelationTrait};
#[derive(Serialize, JsonSchema)]
@ -20,7 +21,7 @@ pub struct ApiPackage {
#[openapi(tag = "test")]
#[get("/search?<query>")]
pub async fn search(query: &str) -> Result<Json<Vec<ApiPackage>>, String> {
match query_aur(query).await {
return match query_aur(query).await {
Ok(v) => {
let mapped = v
.iter()
@ -29,19 +30,19 @@ pub async fn search(query: &str) -> Result<Json<Vec<ApiPackage>>, String> {
version: x.version.clone(),
})
.collect();
return Ok(Json(mapped));
}
Err(e) => {
return Err(format!("{}", e));
}
Ok(Json(mapped))
}
Err(e) => Err(format!("{}", e)),
};
}
#[derive(FromQueryResult, Deserialize, JsonSchema, Serialize)]
#[serde(crate = "rocket::serde")]
pub struct ListPackageModel {
id: i32,
name: String,
count: i32,
status: i32,
}
#[openapi(tag = "test")]
@ -56,6 +57,8 @@ pub async fn package_list(
.select_only()
.column_as(versions::Column::Id.count(), "count")
.column(packages::Column::Name)
.column(packages::Column::Id)
.column(packages::Column::Status)
.group_by(packages::Column::Name)
.into_model::<ListPackageModel>()
.all(db)
@ -64,3 +67,56 @@ pub async fn package_list(
Ok(Json(all))
}
#[openapi(tag = "test")]
#[get("/builds/output?<buildid>")]
pub async fn build_output(
db: &State<DatabaseConnection>,
buildid: i32,
) -> Result<String, NotFound<String>> {
let db = db as &DatabaseConnection;
let build = Builds::find_by_id(buildid)
.one(db)
.await
.map_err(|e| NotFound(e.to_string()))?
.ok_or(NotFound("couldn't find id".to_string()))?;
build.ouput.ok_or(NotFound("No Output".to_string()))
}
#[derive(FromQueryResult, Deserialize, JsonSchema, Serialize)]
#[serde(crate = "rocket::serde")]
pub struct ListBuildsModel {
id: i32,
pkg_id: i32,
version_id: i32,
status: Option<i32>,
}
#[openapi(tag = "test")]
#[get("/builds?<pkgid>")]
pub async fn list_builds(
db: &State<DatabaseConnection>,
pkgid: i32,
) -> Result<Json<Vec<ListBuildsModel>>, NotFound<String>> {
let db = db as &DatabaseConnection;
let build = Builds::find()
.filter(builds::Column::PkgId.eq(pkgid))
.all(db)
.await
.map_err(|e| NotFound(e.to_string()))?;
Ok(Json(
build
.iter()
.map(|x| ListBuildsModel {
id: x.id,
status: x.status,
pkg_id: x.pkg_id,
version_id: x.version_id,
})
.collect::<Vec<_>>(),
))
}

View File

@ -1,6 +1,12 @@
use crate::builder::types::Action;
use crate::db::prelude::{Builds, Packages};
use crate::db::{builds, packages};
use crate::repo::repo::add_pkg;
use sea_orm::{ActiveModelTrait, DatabaseConnection, Set};
use anyhow::anyhow;
use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait, Set};
use std::ops::Add;
use tokio::sync::broadcast;
use tokio::sync::broadcast::error::RecvError;
use tokio::sync::broadcast::Sender;
pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
@ -11,20 +17,65 @@ pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
Action::Build(name, version, url, mut version_model) => {
let db = db.clone();
let build = builds::ActiveModel {
pkg_id: version_model.package_id.clone(),
version_id: version_model.id.clone(),
ouput: Set(None),
status: Set(Some(0)),
..Default::default()
};
let new_build = build.save(&db).await.unwrap();
// spawn new thread for each pkg build
// todo add queue and build two packages in parallel
tokio::spawn(async move {
match add_pkg(url, version, name).await {
let (tx, mut rx) = broadcast::channel::<String>(3);
let db2 = db.clone();
tokio::spawn(async move {
loop {
match rx.recv().await {
Ok(output_line) => {
println!("{output_line}");
let _ = append_db_log_output(
&db2,
output_line,
new_build.id.clone().unwrap(),
)
.await;
}
Err(e) => match e {
RecvError::Closed => {
break;
}
RecvError::Lagged(_) => {}
},
}
}
});
match add_pkg(url, version, name, tx).await {
Ok(pkg_file_name) => {
println!("successfully built package");
let _ = set_pkg_status(
&db,
version_model.package_id.clone().unwrap(),
1,
)
.await;
// update status
version_model.status = Set(Some(1));
version_model.file_name = Set(Some(pkg_file_name));
version_model.update(&db).await.unwrap();
let _ = version_model.update(&db).await;
}
Err(e) => {
version_model.status = Set(Some(2));
version_model.update(&db).await.unwrap();
let _ = set_pkg_status(
&db,
version_model.package_id.clone().unwrap(),
2,
)
.await;
let _ = version_model.update(&db).await;
println!("Error: {e}")
}
@ -35,3 +86,47 @@ pub async fn init(db: DatabaseConnection, tx: Sender<Action>) {
}
}
}
// todo maybe move to helper file
async fn set_pkg_status(
db: &DatabaseConnection,
package_id: i32,
status: i32,
) -> anyhow::Result<()> {
let mut pkg = Packages::find_by_id(package_id)
.one(db)
.await?
.ok_or(anyhow!("no package with id {package_id} found"))?;
pkg.status = status;
let pkg: packages::ActiveModel = pkg.into();
pkg.update(db).await?;
Ok(())
}
async fn append_db_log_output(
db: &DatabaseConnection,
text: String,
build_id: i32,
) -> anyhow::Result<()> {
let build = Builds::find_by_id(build_id)
.one(db)
.await?
.ok_or(anyhow!("build not found"))?;
let mut build: builds::ActiveModel = build.into();
match build.ouput.unwrap() {
None => {
build.ouput = Set(Some(text.add("\n")));
}
Some(s) => {
build.ouput = Set(Some(s.add(text.as_str()).add("\n")));
}
}
build.update(db).await?;
Ok(())
}

View File

@ -26,7 +26,8 @@ create table packages
(
id integer not null
primary key autoincrement,
name text not null
name text not null,
status integer default 0 not null
);
create table status
@ -44,8 +45,7 @@ create table versions
primary key autoincrement,
version TEXT not null,
package_id integer not null,
file_name TEXT,
status INTEGER
file_name TEXT
);
"#,
)

View File

@ -11,6 +11,7 @@ pub struct Model {
#[sea_orm(primary_key)]
pub id: i32,
pub name: String,
pub status: i32,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View File

@ -13,7 +13,6 @@ pub struct Model {
pub version: String,
pub package_id: i32,
pub file_name: Option<String>,
pub status: Option<i32>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View File

@ -1,13 +1,15 @@
use anyhow::anyhow;
use std::fs;
use std::io::{BufRead, BufReader};
use std::process::Command;
use std::process::Stdio;
use std::time::SystemTime;
use tokio::io::{AsyncBufReadExt, BufReader, Lines};
use tokio::sync::broadcast::Sender;
pub fn build_pkgbuild(
pub async fn build_pkgbuild(
folder_path: String,
pkg_vers: &str,
pkg_name: &str,
tx: Sender<String>,
) -> anyhow::Result<String> {
let makepkg = include_str!("../../scripts/makepkg");
@ -15,34 +17,39 @@ pub fn build_pkgbuild(
let script_file = std::env::temp_dir().join("makepkg_custom.sh");
fs::write(&script_file, makepkg).expect("Unable to write script to file");
let mut output = Command::new("bash")
let mut child = tokio::process::Command::new("bash")
.args(&[
script_file.as_os_str().to_str().unwrap(),
"-f",
"--noconfirm",
"-s",
"-c",
"--nocolor",
"-s", // install required deps
"-c", // cleanup leftover files and dirs
"--rmdeps", // remove installed deps with -s
"--noprogressbar", // pacman shouldn't display a progressbar
])
.current_dir(folder_path.clone())
.spawn()
.unwrap();
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
if let Some(stdout) = output.stdout.take() {
let reader = BufReader::new(stdout);
let stderr = child
.stderr
.take()
.ok_or(anyhow!("failed to take stderr"))?;
let stdout = child
.stdout
.take()
.ok_or(anyhow!("failed to take stdout"))?;
// Iterate through each line of output
for line in reader.lines() {
if let Ok(line_content) = line {
// Print the line to the terminal
println!("{}", line_content);
let stderr = BufReader::new(stderr).lines();
let stdout = BufReader::new(stdout).lines();
// todo store line to database for being fetchable from api
}
}
}
let tx1 = tx.clone();
spawn_broadcast_sender(stderr, tx1);
spawn_broadcast_sender(stdout, tx);
// Ensure the command completes
let result = output.wait();
let result = child.wait().await;
match result {
Ok(result) => {
@ -59,6 +66,18 @@ pub fn build_pkgbuild(
locate_built_package(pkg_name.to_string(), pkg_vers.to_string(), folder_path)
}
fn spawn_broadcast_sender<R: tokio::io::AsyncRead + Unpin + Send + 'static>(
mut reader: Lines<BufReader<R>>,
tx: Sender<String>,
) {
tokio::spawn(async move {
while let Ok(Some(line)) = reader.next_line().await {
// println!("directerr: {line}");
let _ = tx.send(line);
}
});
}
fn locate_built_package(
pkg_name: String,
pkg_vers: String,

View File

@ -7,14 +7,25 @@ use anyhow::anyhow;
use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter};
use std::fs;
use std::process::Command;
use tokio::sync::broadcast::Sender;
static REPO_NAME: &str = "repo";
static BASEURL: &str = "https://aur.archlinux.org";
pub async fn add_pkg(url: String, version: String, name: String) -> anyhow::Result<String> {
pub async fn add_pkg(
url: String,
version: String,
name: String,
tx: Sender<String>,
) -> anyhow::Result<String> {
let fname = download_pkgbuild(format!("{}{}", BASEURL, url).as_str(), "./builds").await?;
let pkg_file_name =
build_pkgbuild(format!("./builds/{fname}"), version.as_str(), name.as_str())?;
let pkg_file_name = build_pkgbuild(
format!("./builds/{fname}"),
version.as_str(),
name.as_str(),
tx,
)
.await?;
// todo force overwrite if file already exists
fs::copy(