dsx/dsx_server repo system first implementation

dsx/dsx_server/Cargo.toml

@@ -25,3 +25,4 @@ chrono = "0.4.43"
 tar = "0.4.44"
 flate2 = "1.1.9"
 walkdir = "2.5.0"
+tokio = { version = "1.49.0", features = ["rt"] }

dsx/dsx_server/bacon.toml

@@ -59,7 +59,7 @@ on_success = "back" # so that we don't open the browser at each change
 # if it makes sense for this crate.
 [jobs.run]
 command = [
-    "cargo", "run", "--bin", "dsx_server"
+    "cargo", "run", "--bin", "dsx-server"
     # put launch parameters for your program behind a `--` separator
 ]
 need_stdout = true

dsx/dsx_server/doc/endpoints.md

@@ -12,6 +12,7 @@ GET  /packages/<name>/~repo?q=<query> # search within a package's files
 GET  /packages/<name>/~artifact/      # page listing repo artifacts by date
 GET  /packages/<name>/~artifact/<id>  # page for a specific artifact and status/logs
 
+## API
 POST /api/pkg                         # create repo
 GET  /api/pkg/<name>                  # repo status/metadata
 POST /api/pkg/<name>/push             # upload source tarball

dsx/dsx_server/src/error.rs

@@ -36,3 +36,47 @@ impl From<BuildError> for ApiError {
         }
     }
 }
+
+impl From<DsxError> for ApiError {
+    fn from(err: DsxError) -> Self {
+        ApiError::ServerError(format!("{:?}: {}", err.r#type, err.message))
+    }
+}
+
+#[derive(Debug)]
+pub struct DsxError {
+    pub message: String,
+    pub r#type: ErrorType,
+}
+
+impl DsxError {
+    pub fn new(message: impl AsRef<str>) -> Self {
+        DsxError {
+            message: message.as_ref().to_string(),
+            r#type: ErrorType::Generic,
+        }
+    }
+
+    pub fn with_context(message: impl AsRef<str>, r#type: ErrorType) -> Self {
+        DsxError {
+            message: message.as_ref().to_string(),
+            r#type,
+        }
+    }
+}
+
+impl From<std::io::Error> for DsxError {
+    fn from(err: std::io::Error) -> Self {
+        Self::with_context(err.to_string(), ErrorType::IoError)
+    }
+}
+
+#[derive(Default, Debug)]
+pub enum ErrorType {
+    BuildFailed,
+    IoError,
+
+    #[default]
+    Generic,
+    TarError,
+}

dsx/dsx_server/src/main.rs

@@ -10,7 +10,8 @@ use rocket::data::ToByteUnit;
 use rocket::serde::json::Json;
 use rocket::{fs::FileServer, serde::Deserialize};
 
-use rocket_dyn_templates::{Template, context};
+use rocket_dyn_templates::tera::{Function, Value};
+use rocket_dyn_templates::{Template, context, tera};
 
 use dotenv::dotenv;
 use serde::Serialize;
@@ -23,247 +24,57 @@ use dsx_common::config::DsxConfig;
 mod error;
 mod model;
 
-static DATA_DIR: LazyLock<PathBuf> = LazyLock::new(|| {
-    PathBuf::from(std::env::var("$DATA_DIR").unwrap_or("./data".to_string()))
-});
+mod routes;
 
-// Search for a package
-#[get("/?<q>")]
-fn search_packages(q: Option<String>) -> Result<Template, ApiError> {
-    #[derive(Serialize)]
-    struct Package {
-        name: String,
-        description: String,
-        updated_at: String,
-    }
-
-    let mut packages = Vec::new();
-
-    let dir = match fs::read_dir(DATA_DIR.join("repos")) {
-        Ok(dir) => dir,
-        Err(e) => {
-            warn!("failed to read repos directory: {}", e);
-            return Err(ApiError::InternalServerError(()));
-        }
-    };
-
-    for entry in dir {
-        let entry = entry.map_err(|e| {
-            warn!("failed to read entry: {}", e);
-            ApiError::InternalServerError(())
-        })?;
-
-        let config_path = entry.path().join("repo").join("Dsx.toml");
-        if config_path.exists() {
-            let text = fs::read_to_string(&config_path).map_err(|e| {
-                warn!("failed to read config file: {}", e);
-                ApiError::InternalServerError(())
-            })?;
-            let config: DsxConfig = toml::from_str(&text).map_err(|e| {
-                warn!("failed to parse config file: {}", e);
-                ApiError::InternalServerError(())
-            })?;
-
-            error!("{}", config.description.clone().unwrap_or_default());
-
-            // skip repo if it doesnt match query params
-            if let Some(query) = &q
-                && !(config.name.contains(query)
-                    || config
-                        .description
-                        .clone()
-                        .unwrap_or_default()
-                        .contains(query))
-            {
-                continue;
+fn language_colour() -> impl Function {
+    Box::new(
+        move |args: &std::collections::HashMap<String, Value>| -> tera::Result<Value> {
+            match args.get("lang") {
+                Some(Value::String(lang)) => match lang.as_str() {
+                    // language syntax colour
+                    "dsc" => Ok(Value::String("#000000".to_string())),
+                    "dsa" => Ok(Value::String("#3776AB".to_string())),
+                    _ => Ok(Value::String("#FFFFFF".to_string())),
+                },
+                Some(_) => Err(tera::Error::msg("Invalid argument type")),
+                None => Ok(Value::String("#FFFFFF".to_string())),
             }
-
-            packages.push(Package {
-                name: config.name,
-                description: config.description.unwrap_or_default(),
-                updated_at: String::from("0:00"),
-            })
-        }
-    }
-
-    Ok(Template::render(
-        "packages",
-        context! {
-            packages,
-            query: q.clone().unwrap_or_default()
         },
-    ))
-}
-
-// Main page for a repository, shows status, files, name etc.
-#[get("/<name>")]
-fn package_main(name: &str) -> Result<Template, ApiError> {
-    // get package info
-    let package = Package::load(name)?;
-
-    println!("{}", package.config.name);
-
-    Ok(Template::render(
-        "package_home",
-        context! {
-            parent_path: String::new(),
-            current_path: String::from("/"),
-            package: package,
-        },
-    ))
-}
-
-// Path for a file within a repo
-#[get("/<name>/~repo/<path..>")]
-fn repo_file(name: &str, path: std::path::PathBuf) -> Result<Template, ApiError> {
-    let package = Package::load(name)?;
-
-    Ok(Template::render(
-        "file",
-        context! {
-            package: package,
-        },
-    ))
-}
-
-// Search within a package's files
-#[get("/<name>/~repo?<q>")]
-fn search_repo_files(name: &str, q: Option<String>) -> String {
-    format!("Search within {} for {:?}", name, q)
-}
-
-// Page listing repo artifacts by date
-#[get("/<name>/artifacts")]
-fn list_artifacts(name: &str) -> String {
-    format!("Artifacts for package {}", name)
-}
-
-// Page for a specific artifact and status/logs
-#[get("/<name>/artifacts/<id>")]
-fn artifact_detail(name: &str, id: u64) -> String {
-    format!("Artifact {} details for package {}", id, name)
-}
-
-#[derive(Deserialize)]
-#[serde(crate = "rocket::serde")]
-struct NewRepo<'r> {
-    name: &'r str,
-}
-
-// Create repo
-#[post("/pkg", data = "<repo>")]
-fn create_repo(repo: Json<NewRepo<'_>>) -> Result<(), &'static str> {
-    let path = DATA_DIR.join("repos").join(repo.name);
-
-    if repo.name.is_empty() {
-        return Err("Repository name cannot be empty!");
-    }
-
-    if path.exists() {
-        tracing::info!(
-            "Attempt to create repository '{}' which already exists.",
-            repo.name
-        );
-        return Err("This repository already exists!");
-    }
-
-    if let Err(e) = fs::create_dir_all(path) {
-        tracing::error!(
-            "Attempted to create package with name {} - Error: {e},",
-            repo.name
-        );
-        return Err("Internal server error");
-    }
-
-    Ok(())
-}
-
-// Repo status/metadata
-#[get("/pkg/<name>")]
-fn get_pkg(name: &str) -> Result<Json<(PackageMeta, DsxConfig)>, ApiError> {
-    let package = Package::load(name).map_err(|e| {
-        ApiError::NotFound(String::from("repo with name {name} does not exist"))
-    })?;
-
-    Ok(Json((package.meta, package.config)))
-}
-
-// Upload source tarball
-#[post("/pkg/<name>/push", data = "<data>")]
-async fn push_tarball(name: &str, data: Data<'_>) -> Result<(), ApiError> {
-    let repo_dir = DATA_DIR.join("repos").join(name);
-    let tmp_path = repo_dir.join("upload.tar.gz");
-    let stream = data
-        .open(256.mebibytes())
-        .into_file(&tmp_path)
-        .await
-        .map_err(|e| ApiError::InternalServerError(()))?;
-
-    if !stream.is_complete() {
-        return Err(ApiError::BadRequest("Incomplete upload".to_string()));
-    }
-
-    // Unpack over the existing repo dir.
-    if repo_dir.exists() {
-        fs::remove_dir_all(&repo_dir).map_err(|e| ApiError::InternalServerError(()))?;
-    }
-    fs::create_dir_all(&repo_dir).map_err(|e| ApiError::InternalServerError(()))?;
-
-    let _ = Package::unpack(&tmp_path)?;
-    fs::remove_file(&tmp_path).ok();
-    Ok(())
-}
-
-// Download source tarball
-#[get("/pkg/<name>/pull")]
-fn pull_tarball(
-    name: &str,
-) -> Result<(rocket::http::ContentType, Vec<u8>), Json<ApiError>> {
-    if let Ok(package) = Package::load(name) {
-        let tarball = package.tarball()?;
-        Ok((
-            rocket::http::ContentType::new("application", "octet-stream"),
-            tarball,
-        ))
-    } else {
-        Err(Json(ApiError::NotFound(format!(
-            "repo with name {name} does not exist"
-        ))))
-    }
-}
-
-// Download compiled binary
-#[get("/pkg/<name>/artifact")]
-fn download_artifact(name: &str) -> &'static str {
-    "Download artifact"
+    )
 }
 
 #[launch]
 fn rocket() -> _ {
     dotenv().unwrap();
 
+    use routes::api;
+    use routes::pages;
+
     rocket::build()
         .mount(
             "/packages",
             routes![
-                search_packages,
-                package_main,
-                repo_file,
-                search_repo_files,
-                list_artifacts,
-                artifact_detail,
+                pages::search_packages,
+                pages::package_main,
+                pages::repo_file,
+                pages::search_repo_files,
+                pages::list_artifacts,
+                pages::artifact_detail,
             ],
         )
         .mount(
             "/api",
             routes![
-                create_repo,
-                get_pkg,
-                push_tarball,
-                pull_tarball,
-                download_artifact
+                api::create_repo,
+                api::get_pkg,
+                api::push_tarball,
+                api::pull_tarball,
+                api::download_artifact
             ],
         )
-        .attach(Template::fairing())
+        .attach(Template::custom(|tera| {
+            tera.tera
+                .register_function("language_colour", language_colour());
+        }))
         .mount("/static", FileServer::from("./static"))
 }

dsx/dsx_server/src/model.rs

@@ -1,12 +1,21 @@
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::sync::LazyLock;
 
+use chrono::Utc;
+use dsx_common::builder::{self, BuildContext};
 use dsx_common::config::DsxConfig;
 use serde::{Deserialize, Serialize};
+use walkdir::WalkDir;
 
-use crate::{DATA_DIR, error::ApiError};
+pub static DATA_DIR: LazyLock<PathBuf> = LazyLock::new(|| {
+    PathBuf::from(std::env::var("DATA_DIR").unwrap_or("./data".to_string()))
+});
 
-// stored as a Config.toml above the repository root.
+use crate::error::{ApiError, DsxError, ErrorType};
+
+// stored as a Package.toml above the repository root.
+// not directly modifiable by the user/maintainer.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PackageMeta {
     pub id: String,
@@ -17,11 +26,15 @@ pub struct PackageMeta {
     pub latest_build_status: Option<String>,
     #[serde(default)]
     pub latest_build_id: Option<String>,
+
+    #[serde(default)]
+    pub detected_languages: Option<Vec<String>>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Package {
     pub config: DsxConfig,
+    pub language: String,
     pub meta: PackageMeta,
     pub files: Vec<FileObj>,
 }
@@ -35,47 +48,217 @@ pub struct FileObj {
     pub extension: String,
 }
 
-impl Package {
-    pub fn load(name: &str) -> Result<Self, ApiError> {
-        let repo_path = DATA_DIR.join("repos").join(name);
+enum BuildStatus {
+    // if we don't know the build state, or no build has run before.
+    None,
 
-        let config_contents = fs::read_to_string(repo_path.join("repo/Dsx.toml"))
-            .map_err(|e| {
-                warn!("unable to read config for repo, {e}");
-                ApiError::InternalServerError(())
-            })?;
+    // waiting for build
+    Pending,
 
-        let config: DsxConfig = toml::from_str(&config_contents).map_err(|e| {
-            warn!("Invalid config file for repo! {e}");
-            ApiError::InternalServerError(())
+    // build
+    Building,
+
+    // build result
+    Success,
+    Failure,
+    Error,
+}
+
+pub struct PackageHandle {
+    // id of the package
+    pub package_id: String,
+
+    config: Option<DsxConfig>,
+    meta: Option<PackageMeta>,
+
+    // build info
+    build_state: BuildStatus,
+}
+
+impl PackageHandle {
+    const META_PATH: &'static str = "Package.toml";
+    const CONFIG_PATH: &'static str = "repo/Dsx.toml";
+    const ARTIFACTS_DIR: &'static str = "artifacts";
+
+    pub fn new(package_id: impl Into<String>) -> Self {
+        PackageHandle {
+            package_id: package_id.into(),
+            config: None,
+            meta: None,
+            build_state: BuildStatus::None,
+        }
+    }
+
+    pub fn load(&mut self) -> Result<&mut Self, DsxError> {
+        self.get_config()?;
+        self.get_meta()?;
+        Ok(self)
+    }
+
+    pub fn tarball(&self) -> Result<Vec<u8>, DsxError> {
+        let src_dir = self.path().join("repo");
+        pack_tarball(&src_dir)
+    }
+
+    pub fn unpack(&mut self, archive: &Path) -> Result<&mut Self, DsxError> {
+        let dest = DATA_DIR.join("repos").join(&self.package_id).join("repo");
+        unpack_tarball(archive, &dest)?;
+        self.load()
+    }
+
+    pub fn path(&self) -> PathBuf {
+        DATA_DIR.join("repos").join(&self.package_id)
+    }
+
+    pub fn build(&mut self) -> Result<&mut Self, DsxError> {
+        let ctx = BuildContext {
+            project_dir: self.path().join("repo"),
+            build_dir: self.path().join("build"),
+            artifact_dir: self.path().join("artifacts"),
+        };
+
+        self.build_state = BuildStatus::Building;
+
+        let id = self.package_id.clone();
+        tokio::task::spawn_blocking(move || {
+            let id = id;
+
+            let res = match builder::build_project(ctx) {
+                Ok(_) => "success",
+                Err(_) => "failure",
+            };
+
+            let mut handle = PackageHandle::new(&id);
+            let mut meta = handle.get_meta().unwrap();
+            meta.latest_build_date =
+                Some(Utc::now().format("%Y-%m-%d %H:%M:%S").to_string());
+            meta.latest_build_status = Some(res.to_string());
+            meta.latest_build_id = Some(id);
+            handle.set_meta(meta)
+        });
+
+        Ok(self)
+    }
+
+    pub fn get_config(&mut self) -> Result<DsxConfig, DsxError> {
+        let config_path = self.path().join(Self::CONFIG_PATH);
+
+        let config = fs::read_to_string(&config_path).map_err(|e| {
+            warn!("unable to read Dsx.toml, {e}");
+            DsxError::with_context("Unable to read Dsx.toml", ErrorType::IoError)
         })?;
 
-        let meta_contents =
-            fs::read_to_string(repo_path.join("Package.toml")).map_err(|e| {
-                warn!("unable to read config for repo, {e}");
-                ApiError::InternalServerError(())
-            })?;
-
-        let meta: PackageMeta = toml::from_str(&meta_contents).map_err(|e| {
-            warn!("Invalid meta file for repo! {e}");
-            ApiError::InternalServerError(())
+        let config: DsxConfig = toml::from_str(&config).map_err(|e| {
+            warn!("unable to parse Dsx.toml: {e}");
+            DsxError::new("Unable to parse Dsx.toml")
         })?;
 
-        let dir = fs::read_dir(repo_path.join("repo")).map_err(|e| {
+        self.config = Some(config.clone());
+        Ok(config)
+    }
+
+    pub fn get_meta(&mut self) -> Result<PackageMeta, DsxError> {
+        let meta_path = self.path().join(Self::META_PATH);
+
+        let meta = fs::read_to_string(&meta_path).map_err(|e| {
+            warn!("unable to read Package.toml, {e}");
+            DsxError::with_context("Unable to read Package.toml", ErrorType::IoError)
+        })?;
+
+        let meta: PackageMeta = toml::from_str(&meta).map_err(|e| {
+            warn!("unable to parse Package.toml: {e}");
+            DsxError::new("Unable to parse Package.toml")
+        })?;
+
+        self.meta = Some(meta.clone());
+        Ok(meta)
+    }
+
+    pub fn set_meta(&mut self, meta: PackageMeta) -> Result<(), DsxError> {
+        self.meta = Some(meta);
+        self.save_meta()
+    }
+
+    fn save_meta(&self) -> Result<(), DsxError> {
+        let meta_path = self.path().join(Self::META_PATH);
+        let str = toml::to_string(&self.meta.as_ref().unwrap()).unwrap();
+
+        fs::write(&meta_path, str).map_err(|e| {
+            warn!("unable to write Package.toml, {e}");
+            DsxError::with_context("Unable to write Package.toml", ErrorType::IoError)
+        })?;
+        Ok(())
+    }
+
+    pub fn get_languages(&mut self) -> Result<Vec<String>, DsxError> {
+        self.get_meta()?;
+
+        if let Some(languages) = &self.meta.as_ref().unwrap().detected_languages {
+            return Ok(languages.clone());
+        }
+
+        let mut langs = Vec::new();
+
+        for entry in WalkDir::new(self.path().join("repo/src"))
+            .into_iter()
+            .flatten()
+        {
+            if entry.file_type().is_file() {
+                let path = entry.path();
+                let extension = path.extension().and_then(|ext| ext.to_str());
+                if let Some(ext) = extension {
+                    match ext {
+                        "dsa" | "dsc" => langs.push(ext.to_string()),
+                        _ => {}
+                    }
+                }
+            }
+        }
+
+        self.meta.as_mut().unwrap().detected_languages = Some(langs.clone());
+
+        Ok(langs)
+    }
+
+    pub fn get_artifact(&self) -> Result<Vec<u8>, DsxError> {
+        println!(
+            "{}",
+            self.path()
+                .join(Self::ARTIFACTS_DIR)
+                .join("out.dsb")
+                .display()
+        );
+
+        let artifact_path = self.path().join(Self::ARTIFACTS_DIR).join("out.dsb");
+        fs::read(artifact_path).map_err(|e| {
+            warn!("unable to read artifact for repo: {e}");
+            DsxError::new("Unable to read artifact")
+        })
+    }
+
+    pub fn file_tree(&self, subpath: impl AsRef<Path>) -> Result<Vec<FileObj>, DsxError> {
+        let repo = self.path().join("repo").join(subpath);
+
+        let dir = fs::read_dir(repo).map_err(|e| {
             warn!("unable to read files for repo, {e}");
-            ApiError::InternalServerError(())
+            DsxError::new("Unable to read files in repository")
         })?;
 
         let mut files = Vec::new();
+
+        // for entry in WalkDir::new(repo_path.join("repo")).max_depth(1) {
         for entry in dir {
             let entry = entry.map_err(|e| {
                 warn!("unable to read file entry for repo, {e}");
-                ApiError::InternalServerError(())
+                DsxError::new("Internal error")
             })?;
+
+            // remove root of DATA_DIR/repos/<name>/repo
             let path = entry.path();
+
             let metadata = fs::metadata(&path).map_err(|e| {
                 warn!("unable to read file metadata for repo, {e}");
-                ApiError::InternalServerError(())
+                DsxError::new("Internal error")
             })?;
             let is_dir = metadata.is_dir();
             let size = metadata.len();
@@ -83,7 +266,11 @@ impl Package {
                 .extension()
                 .map_or(String::new(), |ext| ext.to_string_lossy().to_string());
             files.push(FileObj {
-                path: path.to_string_lossy().to_string(),
+                path: path
+                    .strip_prefix(self.path().join("repo"))
+                    .unwrap()
+                    .to_string_lossy()
+                    .to_string(),
                 name: path.file_name().unwrap().to_string_lossy().to_string(),
                 is_dir,
                 size,
@@ -91,28 +278,7 @@ impl Package {
             });
         }
 
-        Ok(Self {
-            config,
-            meta,
-            files,
-        })
-    }
-
-    pub fn unpack(archive: &std::path::Path) -> Result<Self, ApiError> {
-        let repo_name = archive.file_name().unwrap().to_str().unwrap();
-        let dest = DATA_DIR.join("repos").join(repo_name).join("repo");
-        unpack_tarball(archive, &dest)?;
-        let package = Self::load(repo_name)?;
-        Ok(package)
-    }
-
-    pub fn tarball(&self) -> Result<Vec<u8>, ApiError> {
-        let src_dir = self.path().join("repo");
-        pack_tarball(&src_dir)
-    }
-
-    pub fn path(&self) -> PathBuf {
-        DATA_DIR.join("repos").join(&self.meta.id)
+        Ok(files)
     }
 }
 
@@ -126,15 +292,21 @@ use tar::Builder;
 fn unpack_tarball(
     archive: &std::path::Path,
     dest: &std::path::Path,
-) -> Result<(), ApiError> {
-    let file = File::open(archive)?;
+) -> Result<(), DsxError> {
+    let file = File::open(archive)
+        .map_err(|e| DsxError::with_context(e.to_string(), ErrorType::IoError))?;
+
+    fs::create_dir_all(dest)?;
+
     let gz = GzDecoder::new(file);
     let mut tar = tar::Archive::new(gz);
-    tar.unpack(dest)?;
+    tar.unpack(dest)
+        .map_err(|e| DsxError::with_context(e.to_string(), ErrorType::TarError))?;
+
     Ok(())
 }
 
-fn pack_tarball(src_dir: &std::path::Path) -> Result<Vec<u8>, ApiError> {
+fn pack_tarball(src_dir: &std::path::Path) -> Result<Vec<u8>, DsxError> {
     let buf = Vec::new();
     let gz = GzEncoder::new(buf, Compression::default());
     let mut tar = Builder::new(gz);

dsx/dsx_server/src/routes/api.rs

@@ -0,0 +1,119 @@
+use std::fs;
+
+use dsx_common::config::DsxConfig;
+use rocket::{Data, data::ToByteUnit, serde::json::Json};
+use serde::Deserialize;
+
+use crate::{
+    error::ApiError,
+    model::{DATA_DIR, PackageHandle, PackageMeta},
+};
+
+#[derive(Deserialize)]
+#[serde(crate = "rocket::serde")]
+pub struct NewRepo<'r> {
+    name: &'r str,
+}
+
+// Create repo
+#[post("/pkg", data = "<repo>")]
+pub fn create_repo(repo: Json<NewRepo<'_>>) -> Result<(), &'static str> {
+    let path = DATA_DIR.join("repos").join(repo.name);
+
+    if repo.name.is_empty() {
+        return Err("Repository name cannot be empty!");
+    }
+
+    if path.exists() {
+        tracing::info!(
+            "Attempt to create repository '{}' which already exists.",
+            repo.name
+        );
+        return Err("This repository already exists!");
+    }
+
+    if let Err(e) = fs::create_dir_all(path) {
+        tracing::error!(
+            "Attempted to create package with name {} - Error: {e},",
+            repo.name
+        );
+        return Err("Internal server error");
+    }
+
+    Ok(())
+}
+
+// Repo status/metadata
+#[get("/pkg/<id>")]
+pub fn get_pkg(id: &str) -> Result<Json<(PackageMeta, DsxConfig)>, ApiError> {
+    let mut handle = PackageHandle::new(id);
+    let meta = handle.get_meta()?;
+    let config = handle.get_config()?;
+    Ok(Json((meta, config)))
+}
+
+// Upload source tarball
+#[post("/pkg/<id>/push", data = "<data>")]
+pub async fn push_tarball(id: &str, data: Data<'_>) -> Result<(), ApiError> {
+    let mut handle = PackageHandle::new(id);
+    let repo = handle.path();
+
+    let tmp_path = repo.join("upload.tar.gz");
+    let stream = data
+        .open(256.mebibytes())
+        .into_file(&tmp_path)
+        .await
+        .map_err(|e| ApiError::ServerError(e.to_string()))?;
+
+    if !stream.is_complete() {
+        return Err(ApiError::BadRequest("Incomplete upload".to_string()));
+    }
+
+    // Unpack over the existing repo dir.
+    // if the repo is already deleted that's fine so ignore err
+    if handle.path().exists() {
+        let _ = fs::remove_dir_all(repo.join("repo"));
+    }
+
+    handle.unpack(&tmp_path)?;
+    fs::remove_file(&tmp_path).ok();
+
+    // we don't care if there's an error in the build.
+    let _ = handle.build();
+
+    Ok(())
+}
+
+// Download source tarball
+#[get("/pkg/<id>/pull")]
+pub fn pull_tarball(
+    id: &str,
+) -> Result<(rocket::http::ContentType, Vec<u8>), Json<ApiError>> {
+    if let Ok(tarball) = PackageHandle::new(id).tarball() {
+        Ok((
+            rocket::http::ContentType::new("application", "octet-stream"),
+            tarball,
+        ))
+    } else {
+        Err(Json(ApiError::NotFound(format!(
+            "repo with id {id} does not exist"
+        ))))
+    }
+}
+
+// Download compiled binary
+#[get("/pkg/<id>/artifact")]
+pub fn download_artifact(
+    id: &str,
+) -> Result<(rocket::http::ContentType, Vec<u8>), Json<ApiError>> {
+    if let Ok(artifact) = PackageHandle::new(id).get_artifact() {
+        Ok((
+            rocket::http::ContentType::new("application", "octet-stream"),
+            artifact,
+        ))
+    } else {
+        Err(Json(ApiError::NotFound(format!(
+            "repo with id {id} does not have a valid artifact"
+        ))))
+    }
+}

dsx/dsx_server/src/routes/mod.rs

@@ -0,0 +1,2 @@
+pub mod api;
+pub mod pages;

dsx/dsx_server/src/routes/pages.rs

@@ -0,0 +1,145 @@
+use std::{fs, path::Path};
+
+use dsx_common::config::DsxConfig;
+use rocket_dyn_templates::{Template, context};
+use serde::Serialize;
+
+use crate::{
+    error::ApiError,
+    model::{DATA_DIR, PackageHandle},
+};
+
+// Search for a package
+#[get("/?<q>")]
+pub fn search_packages(q: Option<String>) -> Result<Template, ApiError> {
+    #[derive(Serialize)]
+    struct Package {
+        name: String,
+        description: String,
+        updated_at: String,
+    }
+
+    let mut packages = Vec::new();
+
+    let dir = match fs::read_dir(DATA_DIR.join("repos")) {
+        Ok(dir) => dir,
+        Err(e) => {
+            warn!("failed to read repos directory: {}", e);
+            return Err(ApiError::InternalServerError(()));
+        }
+    };
+
+    for entry in dir {
+        let entry = entry.map_err(|e| {
+            warn!("failed to read entry: {}", e);
+            ApiError::InternalServerError(())
+        })?;
+
+        let config_path = entry.path().join("repo").join("Dsx.toml");
+        if config_path.exists() {
+            let text = fs::read_to_string(&config_path).map_err(|e| {
+                warn!("failed to read config file: {}", e);
+                ApiError::InternalServerError(())
+            })?;
+            let config: DsxConfig = toml::from_str(&text).map_err(|e| {
+                warn!("failed to parse config file: {}", e);
+                ApiError::InternalServerError(())
+            })?;
+
+            error!("{}", config.description.clone().unwrap_or_default());
+
+            // skip repo if it doesnt match query params
+            if let Some(query) = &q
+                && !(config.name.contains(query)
+                    || config
+                        .description
+                        .clone()
+                        .unwrap_or_default()
+                        .contains(query))
+            {
+                continue;
+            }
+
+            packages.push(Package {
+                name: config.name,
+                description: config.description.unwrap_or_default(),
+                updated_at: String::from("0:00"),
+            })
+        }
+    }
+
+    Ok(Template::render(
+        "packages",
+        context! {
+            packages,
+            query: q.clone().unwrap_or_default()
+        },
+    ))
+}
+
+// Main page for a repository, shows status, files, name etc.
+#[get("/<id>")]
+pub fn package_main(id: &str) -> Result<Template, ApiError> {
+    // get package info
+
+    let mut handle = PackageHandle::new(id);
+    let meta = handle.get_meta()?;
+    let config = handle.get_config()?;
+    let files = handle.file_tree("")?;
+
+    let parent_path: Option<String> = None;
+    let current_path: Option<String> = None;
+
+    Ok(Template::render(
+        "package_home",
+        context! {
+            parent_path,
+            current_path,
+            meta: meta,
+            config: config,
+            files: files,
+        },
+    ))
+}
+
+// Path for a file within a repo
+#[get("/<id>/~repo/<path..>", rank = 1)]
+pub fn repo_file(id: &str, path: std::path::PathBuf) -> Result<Template, ApiError> {
+    // get package info
+    let mut handle = PackageHandle::new(id);
+    let meta = handle.get_meta()?;
+    let config = handle.get_config()?;
+    let files = handle.file_tree(&path)?;
+
+    let parent_path = path.parent().unwrap_or(Path::new(""));
+    let current_path = &path;
+
+    Ok(Template::render(
+        "package_home",
+        context! {
+            parent_path,
+            current_path,
+            meta: meta,
+            config: config,
+            files: files,
+        },
+    ))
+}
+
+// Search within a package's files
+#[get("/<name>/~repo?<q>", rank = 2)]
+pub fn search_repo_files(name: &str, q: Option<String>) -> String {
+    format!("Search within {} for {:?}", name, q)
+}
+
+// Page listing repo artifacts by date
+#[get("/<name>/artifacts")]
+pub fn list_artifacts(name: &str) -> String {
+    format!("Artifacts for package {}", name)
+}
+
+// Page for a specific artifact and status/logs
+#[get("/<name>/artifacts/<id>")]
+pub fn artifact_detail(name: &str, id: u64) -> String {
+    format!("Artifact {} details for package {}", id, name)
+}

dsx/dsx_server/templates/components/file_tree.html.tera

@@ -4,17 +4,17 @@
          <div class="file-toolbar">
              <div class="file-toolbar-search">
                  <svg class="file-toolbar-search-icon" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/></svg>
-                 <form method="get" action="/packages/{{ package.config.name }}/~repo">
+                 <form method="get" action="/packages/{{ meta.id }}/~repo">
                      <input type="text" name="q" placeholder="Filter files…" value="{{ query | default(value='') }}">
                  </form>
              </div>
              <span class="dim small mono">{{ current_path | default(value="/") }}</span>
          </div>
 
-         {% if package.files and package.files | length > 0 %}
+         {% if files and files | length > 0 %}
              <div class="file-tree-body">
                  {% if current_path %}
-                 <a class="file-row" href="/packages/{{ package.config.name }}/~repo/{{ parent_path | default(value='') }}">
+                 <a class="file-row" href="/packages/{{ meta.id }}/~repo/{{ parent_path | default(value='') }}">
                      <span class="file-icon">
                          <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M3 9l9-7 9 7v11a2 2 0 01-2 2H5a2 2 0 01-2-2z"/></svg>
                      </span>
@@ -23,9 +23,8 @@
                  {% endif %}
 
 
-
-                 {% for file in package.files %}
-                     {{ files::file_view(package = package.config.name, file = file) }}
+                 {% for file in files %}
+                     {{ files::file_view(package = meta.id, file = file) }}
                  {% endfor %}
              </div>
          {% else %}

dsx/dsx_server/templates/components/sidebar.html.tera

@@ -6,25 +6,28 @@
         <div class="sidebar-card-title">Latest Build</div>
         <div class="sidebar-card-body">
             <div class="build-status-row">
-                {% if package.meta.latest_build_status == "success" %}
+                {% if meta.latest_build_status == "success" %}
                     <span class="badge badge-success"><span class="dot"></span>passing</span>
-                {% elif package.meta.latest_build_status == "failure" %}
+                {% elif meta.latest_build_status == "failure" %}
                     <span class="badge badge-failure"><span class="dot"></span>failing</span>
-                {% elif package.meta.latest_build_status == "running" %}
+                {% elif meta.latest_build_status == "running" %}
                     <span class="badge badge-running"><span class="dot dot-pulse"></span>running</span>
-                {% elif package.meta.latest_build_status == "pending" %}
+                {% elif meta.latest_build_status == "pending" %}
                     <span class="badge badge-pending"><span class="dot"></span>pending</span>
                 {% else %}
                     <span class="dim small">No builds yet</span>
                 {% endif %}
             </div>
-            {% if package.meta.latest_build_id %}
-            <a class="build-link" href="/packages/{{ package.config.name }}/~artifact/{{ package.meta.latest_build_id }}">
-                #{{ package.meta.latest_build_id }} — view logs →
+            {% if meta.latest_build_id %}
+            <a class="build-link"
+                href="/api/pkg/{{ config.name }}/artifact"
+                download="{{ config.name }}-{{ meta.latest_build_date }}.dsb"
+                style="font-size:13px">
+                "{{ config.name }}-{{ meta.latest_build_date }}.dsb"
             </a>
             {% endif %}
-            {% if package.meta.latest_build_date %}
-            <div class="dim small mono" style="margin-top:6px">{{ package.meta.latest_build_date }}</div>
+            {% if meta.latest_build_date %}
+            <div class="dim small mono" style="margin-top:6px">{{ meta.latest_build_date }}</div>
             {% endif %}
         </div>
     </div>
@@ -67,11 +70,11 @@
     <div class="card sidebar-card">
         <div class="sidebar-card-title">Links</div>
         <div class="sidebar-card-body" style="display:flex;flex-direction:column;gap:6px">
-            <a href="/packages/{{ package.config.name }}/~repo" style="font-size:13px">
+            <a href="/packages/{{ config.name }}/~repo" style="font-size:13px">
                 <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" style="vertical-align:middle;margin-right:4px"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><polyline points="14 2 14 8 20 8"/></svg>
                 Browse files
             </a>
-            <a href="/packages/{{ package.config.name }}/~artifact/" style="font-size:13px">
+            <a href="/packages/{{ config.name }}/~artifact/" style="font-size:13px">
                 <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" style="vertical-align:middle;margin-right:4px"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg>
                 Build history
             </a>

dsx/dsx_server/templates/package_home.html.tera

@@ -1,7 +1,7 @@
 {% extends "base" %}
 {% import 'components/file' as files %}
 
-{% block title %}{{ package.config.name }}{% endblock %}
+{% block title %}{{ config.name }}{% endblock %}
 
 {% block extra_head %}
 <style>
@@ -171,24 +171,24 @@
     <div class="breadcrumb">
         <a href="/packages/">packages</a>
         <span class="breadcrumb-sep">/</span>
-        <span>{{ package.config.name }}</span>
+        <span>{{ config.name }}</span>
     </div>
 
     <div class="pkg-header">
         <h1>
-            {{ package.config.name }}
-            {% if package.meta.latest_build_status == "success" %}
+            {{ config.name }}
+            {% if meta.latest_build_status == "success" %}
                 <span class="badge badge-success"><span class="dot"></span>passing</span>
-            {% elif package.meta.latest_build_status == "failure" %}
+            {% elif meta.latest_build_status == "failure" %}
                 <span class="badge badge-failure"><span class="dot"></span>failing</span>
-            {% elif package.meta.latest_build_status == "running" %}
+            {% elif meta.latest_build_status == "running" %}
                 <span class="badge badge-running"><span class="dot dot-pulse"></span>running</span>
-            {% elif package.meta.latest_build_status == "pending" %}
+            {% elif meta.latest_build_status == "pending" %}
                 <span class="badge badge-pending"><span class="dot"></span>pending</span>
             {% endif %}
         </h1>
-        {% if package.config.description %}
-        <p class="pkg-desc">{{ package.config.description }}</p>
+        {% if config.description %}
+        <p class="pkg-desc">{{ config.description }}</p>
         {% endif %}
     </div>