Set AVAILABLE_PARALLELISM at build and run time

buildpacks/nodejs-engine/CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Set `HEROKU_AVAILABLE_PARALLELISM` environment variable at build and run time
+
 ## [3.3.3] - 2024-11-22
 
 ### Added
@@ -714,56 +716,109 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add shellcheck to test suite ([#24](https://github.com/heroku/nodejs-engine-buildpack/pull/24))
 
 [unreleased]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.3...HEAD
+
 [3.3.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.2...v3.3.3
+
 [3.3.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.1...v3.3.2
+
 [3.3.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.0...v3.3.1
+
 [3.3.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.18...v3.3.0
+
 [3.2.18]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.17...v3.2.18
+
 [3.2.17]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.16...v3.2.17
+
 [3.2.16]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.15...v3.2.16
+
 [3.2.15]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.14...v3.2.15
+
 [3.2.14]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.13...v3.2.14
+
 [3.2.13]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.12...v3.2.13
+
 [3.2.12]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.11...v3.2.12
+
 [3.2.11]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.10...v3.2.11
+
 [3.2.10]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.9...v3.2.10
+
 [3.2.9]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.8...v3.2.9
+
 [3.2.8]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.7...v3.2.8
+
 [3.2.7]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.6...v3.2.7
+
 [3.2.6]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.5...v3.2.6
+
 [3.2.5]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.4...v3.2.5
+
 [3.2.4]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.3...v3.2.4
+
 [3.2.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.2...v3.2.3
+
 [3.2.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.1...v3.2.2
+
 [3.2.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.0...v3.2.1
+
 [3.2.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.1.0...v3.2.0
+
 [3.1.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.6...v3.1.0
+
 [3.0.6]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.5...v3.0.6
+
 [3.0.5]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.4...v3.0.5
+
 [3.0.4]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.3...v3.0.4
+
 [3.0.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.2...v3.0.3
+
 [3.0.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.1...v3.0.2
+
 [3.0.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.0...v3.0.1
+
 [3.0.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.6...v3.0.0
+
 [2.6.6]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.5...v2.6.6
+
 [2.6.5]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.4...v2.6.5
+
 [2.6.4]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.3...v2.6.4
+
 [2.6.3]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.2...v2.6.3
+
 [2.6.2]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.1...v2.6.2
+
 [2.6.1]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.0...v2.6.1
+
 [2.6.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.5.0...v2.6.0
+
 [2.5.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.4.1...v2.5.0
+
 [2.4.1]: https://github.com/heroku/buildpacks-nodejs/compare/v2.4.0...v2.4.1
+
 [2.4.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.3.0...v2.4.0
+
 [2.3.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.2.0...v2.3.0
+
 [2.2.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.1.0...v2.2.0
+
 [2.1.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.0.0...v2.1.0
+
 [2.0.0]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.7...v2.0.0
+
 [1.1.7]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.6...v1.1.7
+
 [1.1.6]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.5...v1.1.6
+
 [1.1.5]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.4...v1.1.5
+
 [1.1.4]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.3...v1.1.4
+
 [1.1.3]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.2...v1.1.3
+
 [1.1.2]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.1...v1.1.2
+
 [1.1.1]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.0...v1.1.1
+
 [1.1.0]: https://github.com/heroku/buildpacks-nodejs/releases/tag/v1.1.0

buildpacks/nodejs-engine/Cargo.toml

@@ -17,7 +17,9 @@ thiserror = "1"
 toml = "0.8"
 
 [dev-dependencies]
+indoc = "2"
 libcnb-test = "=0.25.0"
+regex = "1"
 serde_json = "1"
 test_support.workspace = true
 ureq = "2"

buildpacks/nodejs-engine/src/bin/available_parallelism.rs

@@ -0,0 +1,17 @@
+// Required due to: https://github.com/rust-lang/rust/issues/95513
+#![allow(unused_crate_dependencies)]
+
+use heroku_nodejs_utils::available_parallelism::available_parallelism_env;
+use libcnb::data::exec_d::ExecDProgramOutputKey;
+use libcnb::exec_d::write_exec_d_program_output;
+use std::collections::HashMap;
+
+fn main() {
+    let mut output: HashMap<ExecDProgramOutputKey, String> = HashMap::with_capacity(1);
+    let (available_parallelism_env_key, available_parallelism_env_value) =
+        available_parallelism_env();
+    if let Ok(exec_d_output_key) = available_parallelism_env_key.parse::<ExecDProgramOutputKey>() {
+        output.insert(exec_d_output_key, available_parallelism_env_value);
+    }
+    write_exec_d_program_output(output);
+}

buildpacks/nodejs-engine/src/configure_available_parallelism.rs

@@ -0,0 +1,38 @@
+use crate::{NodeJsEngineBuildpack, NodeJsEngineBuildpackError};
+use heroku_nodejs_utils::available_parallelism::available_parallelism_env;
+use libcnb::additional_buildpack_binary_path;
+use libcnb::build::BuildContext;
+use libcnb::data::layer_name;
+use libcnb::layer::UncachedLayerDefinition;
+use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope};
+
+pub(crate) fn configure_available_parallelism(
+    context: &BuildContext<NodeJsEngineBuildpack>,
+) -> Result<(), libcnb::Error<NodeJsEngineBuildpackError>> {
+    let available_parallelism_layer = context.uncached_layer(
+        layer_name!("available_parallelism"),
+        UncachedLayerDefinition {
+            build: true,
+            launch: true,
+        },
+    )?;
+
+    let (available_parallelism_env_key, available_parallelism_env_value) =
+        available_parallelism_env();
+
+    // set for the build time env (for webpack plugins or other tools that spin up processes)
+    available_parallelism_layer.write_env(LayerEnv::new().chainable_insert(
+        Scope::Build,
+        ModificationBehavior::Override,
+        available_parallelism_env_key,
+        available_parallelism_env_value,
+    ))?;
+
+    // set for the run time env
+    available_parallelism_layer.write_exec_d_programs([(
+        "available_parallelism",
+        additional_buildpack_binary_path!("available_parallelism"),
+    )])?;
+
+    Ok(())
+}

buildpacks/nodejs-engine/src/main.rs

@@ -1,10 +1,13 @@
 use std::env::consts;
 
 use crate::attach_runtime_metrics::{attach_runtime_metrics, NodeRuntimeMetricsError};
+use crate::configure_available_parallelism::configure_available_parallelism;
 use crate::configure_web_env::configure_web_env;
 use crate::install_node::{install_node, DistLayerError};
 use heroku_nodejs_utils::package_json::{PackageJson, PackageJsonError};
 use heroku_nodejs_utils::vrs::{Requirement, Version};
+#[cfg(test)]
+use indoc as _;
 use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder};
 use libcnb::data::build_plan::BuildPlanBuilder;
 use libcnb::data::launch::{LaunchBuilder, ProcessBuilder};
@@ -19,6 +22,8 @@ use libherokubuildpack::inventory::artifact::{Arch, Os};
 use libherokubuildpack::inventory::Inventory;
 use libherokubuildpack::log::{log_error, log_header, log_info};
 #[cfg(test)]
+use regex as _;
+#[cfg(test)]
 use serde_json as _;
 use sha2::Sha256;
 #[cfg(test)]
@@ -28,6 +33,7 @@ use thiserror::Error;
 use ureq as _;
 
 mod attach_runtime_metrics;
+mod configure_available_parallelism;
 mod configure_web_env;
 mod install_node;
 
@@ -108,6 +114,7 @@ impl Buildpack for NodeJsEngineBuildpack {
         install_node(&context, target_artifact)?;
 
         configure_web_env(&context)?;
+        configure_available_parallelism(&context)?;
 
         if Requirement::parse(MINIMUM_NODE_VERSION_FOR_METRICS)
             .expect("should be a valid version range")

buildpacks/nodejs-engine/tests/integration_test.rs

@@ -1,11 +1,16 @@
 // Required due to: https://github.com/rust-lang/rust/issues/95513
 #![allow(unused_crate_dependencies)]
 
-use libcnb_test::{assert_contains, assert_not_contains, ContainerConfig};
+use indoc::indoc;
+use libcnb::data::buildpack_id;
+use libcnb_test::{
+    assert_contains, assert_contains_match, assert_not_contains, BuildpackReference,
+    ContainerConfig,
+};
 use std::time::Duration;
 use test_support::{
-    assert_web_response, nodejs_integration_test, nodejs_integration_test_with_config,
-    set_node_engine, wait_for, PORT,
+    assert_web_response, custom_buildpack, integration_test_with_config, nodejs_integration_test,
+    nodejs_integration_test_with_config, set_node_engine, wait_for, PORT,
 };
 
 const APPLICATION_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
@@ -67,6 +72,34 @@ fn reinstalls_node_if_version_changes() {
     );
 }
 
+#[test]
+#[ignore]
+fn heroku_available_parallelism_is_set_at_build_and_runtime() {
+    integration_test_with_config(
+        "./fixtures/node-with-indexjs",
+        |_| {},
+        |ctx| {
+            assert_contains_match!(ctx.pack_stdout, "HEROKU_AVAILABLE_PARALLELISM=\\d+");
+            assert_contains_match!(
+                ctx.run_shell_command("env").stdout,
+                "HEROKU_AVAILABLE_PARALLELISM=\\d+"
+            );
+        },
+        &[
+            BuildpackReference::WorkspaceBuildpack(buildpack_id!("heroku/nodejs")),
+            BuildpackReference::Other(
+                custom_buildpack()
+                    .id("test/echo-build-parallelism")
+                    .build(indoc! { "
+                        #!/usr/bin/env bash
+                        env
+                    " })
+                    .call(),
+            ),
+        ],
+    );
+}
+
 #[test]
 #[ignore]
 fn runtime_metrics_script_is_activated_when_heroku_metrics_url_is_set() {

common/nodejs-utils/src/available_parallelism.rs

@@ -0,0 +1,16 @@
+pub const HEROKU_AVAILABLE_PARALLELISM: &str = "HEROKU_AVAILABLE_PARALLELISM";
+
+#[must_use]
+pub fn available_parallelism_env() -> (String, String) {
+    (
+        HEROKU_AVAILABLE_PARALLELISM.to_string(),
+        std::thread::available_parallelism()
+            // XXX: The Rust implementation always rounds down the value reported here if the
+            //      (quota / period) calculated from cgroups cpu.max produces a fractional value.
+            //      For Heroku Fir Dynos this will always end up reducing the cpu allocation
+            //      value by 1 since a small amount of quota is reserved for the system so we need
+            //      to add that back unless Rust changes how they deal with rounding.
+            .map(|value| (value.get() + 1).to_string())
+            .unwrap_or_default(),
+    )
+}

common/nodejs-utils/src/lib.rs

@@ -2,6 +2,7 @@ use keep_a_changelog_file as _;
 use sha2 as _;
 
 pub mod application;
+pub mod available_parallelism;
 pub mod buildplan;
 pub mod distribution;
 pub mod inv;

test_support/src/lib.rs

@@ -188,7 +188,11 @@ pub fn add_package_json_dependency(app_dir: &Path, package_name: &str, package_v
 
 pub fn add_build_script(app_dir: &Path, script: &str) {
     update_package_json(app_dir, |json| {
-        let scripts = json["scripts"].as_object_mut().unwrap();
+        let scripts = json
+            .entry("scripts")
+            .or_insert(serde_json::Value::Object(serde_json::Map::new()))
+            .as_object_mut()
+            .unwrap();
         scripts.insert(
             script.to_string(),
             serde_json::Value::String(format!("echo 'executed {script}'")),