Merge branch 'main' into add-tests

Author: delsner

.gitattributes

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @pavelzw @delsner @0xbe7a

.github/workflows/ci.yml

@@ -7,16 +7,58 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Run tests
+  pre-commit-checks:
+    name: Pre-commit Checks
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
+        with:
+          environments: default lint
+      - name: Set up cargo cache # use https://github.com/Swatinem/rust-cache once whitelisted
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+      - name: pre-commit
+        run: pixi run pre-commit-run --color=always --show-diff-on-failure
+
+  unit-tests:
+    name: test
+    timeout-minutes: 30
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        toolchain: [stable]
     steps:
-      - uses: actions/checkout@v4
-      - run: rustup update ${{ matrix.toolchain }}
-      - run: rustup default ${{ matrix.toolchain }}
-      - run: cargo build --verbose
-      - run: cargo test --verbose
+      - name: Checkout branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659
+      - name: Set up cargo cache # use https://github.com/Swatinem/rust-cache once whitelisted
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+      - name: Run test
+        run: pixi run test

.gitignore

@@ -120,3 +120,6 @@ env/
 unpack/
 cache/
 activate.*
+# pixi environments
+.pixi
+*.egg-info

.pre-commit-config.yaml

@@ -0,0 +1,55 @@
+repos:
+  - repo: local
+    hooks:
+      # prettier
+      - id: prettier
+        name: prettier
+        entry: pixi run -e lint prettier --write --list-different --ignore-unknown
+        language: system
+        types: [text]
+        files: \.(md|yml|yaml)$
+      # pre-commit-hooks
+      - id: trailing-whitespace-fixer
+        name: trailing-whitespace-fixer
+        entry: pixi run -e lint trailing-whitespace-fixer
+        language: system
+        types: [text]
+      - id: end-of-file-fixer
+        name: end-of-file-fixer
+        entry: pixi run -e lint end-of-file-fixer
+        language: system
+        types: [text]
+      - id: check-merge-conflict
+        name: check-merge-conflict
+        entry: pixi run -e lint check-merge-conflict --assume-in-merge
+        language: system
+        types: [text]
+      # typos
+      - id: typos
+        name: typos
+        entry: pixi run -e lint typos --force-exclude
+        language: system
+        types: [text]
+        require_serial: true
+      # cargo fmt and clippy
+      - id: cargo-fmt-conda
+        name: cargo-fmt-conda
+        description: "Run `cargo fmt` for formatting rust sources."
+        entry: pixi run -e default cargo fmt --
+        language: system
+        require_serial: false
+        types: [rust]
+      - id: clippy-conda
+        name: clippy-conda
+        description: "Run `clippy` to lint rust sources."
+        entry: pixi run -e default cargo clippy --all-targets --all-features --workspace -- -D warnings
+        pass_filenames: false
+        language: system
+        require_serial: false
+        types: [rust]
+      # taplo
+      - id: taplo
+        name: taplo
+        entry: pixi run -e lint taplo format
+        language: system
+        types: [toml]

.vscode/launch.json

@@ -90,4 +90,4 @@
             "cwd": "${workspaceFolder}"
         }
     ]
-}
+}

pixi.lock

@@ -0,0 +1,27 @@
+[project]
+name = "pixi-pack"
+version = "0.1.0"
+description = "A command line tool to pack and unpack conda environments for easy sharing."
+channels = ["conda-forge"]
+platforms = ["osx-arm64", "osx-64", "linux-64", "linux-aarch64", "win-64"]
+
+[tasks]
+build = "cargo build --release"
+test = "cargo test"
+
+[dependencies]
+rust = "1.77.2"
+
+[feature.lint.dependencies]
+pre-commit = "*"
+prettier = "*"
+taplo = "*"
+pre-commit-hooks = "*"
+typos = "*"
+[feature.lint.tasks]
+pre-commit-install = "pre-commit install"
+pre-commit-run = "pre-commit run -a"
+
+[environments]
+default = ["lint"]
+lint = ["lint"]

pixi.toml

@@ -15,8 +15,8 @@ use futures::{
     StreamExt, TryStreamExt,
 };
 use indicatif::ProgressStyle;
-use rattler_conda_types::Platform;
-use rattler_lock::{LockFile, Package};
+use rattler_conda_types::{PackageRecord, Platform};
+use rattler_lock::{CondaPackage, LockFile, Package};
 use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage};
 use reqwest_middleware::ClientWithMiddleware;
 use tokio_tar::Builder;
@@ -65,9 +65,20 @@ pub async fn pack(options: PackOptions) -> Result<()> {
         options.platform.as_str()
     ))?;
 
+    let mut conda_packages = Vec::new();
+
+    for package in packages {
+        match package {
+            Package::Conda(p) => conda_packages.push(p),
+            Package::Pypi(_) => {
+                anyhow::bail!("pypi packages are not supported in pixi-pack");
+            }
+        }
+    }
+
     // Download packages to temporary directory.
-    tracing::info!("Downloading {} packages", packages.len());
-    let bar = indicatif::ProgressBar::new(packages.len() as u64);
+    tracing::info!("Downloading {} packages", conda_packages.len());
+    let bar = indicatif::ProgressBar::new(conda_packages.len() as u64);
     bar.set_style(
         ProgressStyle::with_template(
             "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
@@ -81,7 +92,7 @@ pub async fn pack(options: PackOptions) -> Result<()> {
 
     let channel_dir = output_folder.path().join(CHANNEL_DIRECTORY_NAME);
 
-    stream::iter(packages)
+    stream::iter(conda_packages.iter())
         .map(Ok)
         .try_for_each_concurrent(50, |package| async {
             download_package(&client, package, &channel_dir).await?;
@@ -105,6 +116,13 @@ pub async fn pack(options: PackOptions) -> Result<()> {
     let metadata = serde_json::to_string_pretty(&options.metadata)?;
     metadata_file.write_all(metadata.as_bytes()).await?;
 
+    // Create environment file.
+    create_environment_file(
+        output_folder.path(),
+        conda_packages.iter().map(|p| p.package_record()),
+    )
+    .await?;
+
     // Pack = archive + compress the contents.
     archive_directory(output_folder.path(), &options.output_file, options.level)
         .await
@@ -143,25 +161,21 @@ fn reqwest_client_from_auth_storage(auth_file: Option<PathBuf>) -> Result<Client
 /// Download a conda package to a given output directory.
 async fn download_package(
     client: &ClientWithMiddleware,
-    package: Package,
+    package: &CondaPackage,
     output_dir: &Path,
 ) -> Result<()> {
-    let conda_package = package
-        .as_conda()
-        .ok_or(anyhow!("package is not a conda package"))?; // TODO: we might want to skip here
-
-    let output_dir = output_dir.join(&conda_package.package_record().subdir);
+    let output_dir = output_dir.join(&package.package_record().subdir);
     create_dir_all(&output_dir)
         .await
         .map_err(|e| anyhow!("could not create download directory: {}", e))?;
 
-    let file_name = conda_package
+    let file_name = package
         .file_name()
         .ok_or(anyhow!("could not get file name"))?;
     let mut dest = File::create(output_dir.join(file_name)).await?;
 
-    tracing::debug!("Fetching package {}", conda_package.url());
-    let mut response = client.get(conda_package.url().to_string()).send().await?;
+    tracing::debug!("Fetching package {}", package.url());
+    let mut response = client.get(package.url().to_string()).send().await?;
 
     while let Some(chunk) = response.chunk().await? {
         dest.write_all(&chunk).await?;
@@ -208,3 +222,34 @@ async fn archive_directory(
 
     Ok(())
 }
+
+async fn create_environment_file(
+    destination: &Path,
+    packages: impl IntoIterator<Item = &PackageRecord>,
+) -> Result<()> {
+    let environment_path = destination.join("environment.yml");
+
+    let mut environment = String::new();
+
+    environment.push_str("channels:\n");
+    environment.push_str(&format!("  - ./{CHANNEL_DIRECTORY_NAME}\n",));
+    environment.push_str("  - nodefaults\n");
+    environment.push_str("dependencies:\n");
+
+    for package in packages {
+        let match_spec_str = format!(
+            "{}={}={}",
+            package.name.as_normalized(),
+            package.version,
+            package.build,
+        );
+
+        environment.push_str(&format!("  - {}\n", match_spec_str));
+    }
+
+    fs::write(environment_path, environment)
+        .await
+        .map_err(|e| anyhow!("Could not write environment file: {}", e))?;
+
+    Ok(())
+}

src/pack.rs

@@ -138,9 +138,9 @@ async fn unarchive(archive_path: &Path, target_dir: &Path) -> Result<()> {
 
     let reader = tokio::io::BufReader::new(file);
 
-    let decocder = ZstdDecoder::new(reader);
+    let decoder = ZstdDecoder::new(reader);
 
-    let mut archive = Archive::new(decocder);
+    let mut archive = Archive::new(decoder);
 
     archive
         .unpack(target_dir)