Q# Package System (microsoft#1698)

This PR:

1. Refactors the compiler API such that the caller passes in a
constructed package store, allowing for dependencies to be pre-compiled
and inserted before compiling user code
2. Introduces the `BuildableProgram` abstraction, which encapsulates the
notion of "some user code that is ready to compile, along with a package
store that has all of its dependencies compiled"
3. As a drive-by fix, fixes circuits for internal operations
4. Removes the notion of `Visibility` from the AST, since visibility is
calculated via exports now. `internal` is still parsed without error for
backwards compatibility, but is a no-op because `internal` is the
default now.
5. Enforces visibility in the HIR, where items must be
`Visibility::Public` to show up across packages as a `GlobalItem`
6. Adds export statements to the standard library to preserve the
existing API with the new internal-by-default semantics
7. Inserts all packages into the user code's namespace tree with the
prefix defined as the package alias in the `qsharp.json`



closes microsoft#883 


The Q# formatting CI stage will fail until microsoft#1692 goes in

This PR supersedes microsoft#1693

.prettierignore

@@ -24,4 +24,5 @@ __pycache__/
 /target/
 /vscode/out/
 /vscode/test/out/
+/vscode/test/**/test-workspace/
 /wasm/

build.py

@@ -488,6 +488,7 @@ def run_python_integration_tests(cwd, interpreter):
             or f.startswith("circuits.")
             or f.startswith("iterative_phase_estimation.")
             or f.startswith("repeat_until_success.")
+            or f.startswith("python-deps.")
         )
     ]
     python_bin = use_python_env(samples_src)

compiler/qsc/benches/eval.rs

@@ -18,12 +18,14 @@ const ARRAY_LITERAL: &str = include_str!("./array_literal");
 pub fn teleport(c: &mut Criterion) {
     c.bench_function("Teleport evaluation", |b| {
         let sources = SourceMap::new([("Teleportation.qs".into(), TELEPORT.into())], None);
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
         b.iter(move || {
@@ -37,12 +39,14 @@ pub fn teleport(c: &mut Criterion) {
 pub fn deutsch_jozsa(c: &mut Criterion) {
     c.bench_function("Deutsch-Jozsa evaluation", |b| {
         let sources = SourceMap::new([("DeutschJozsa.qs".into(), DEUTSCHJOZSA.into())], None);
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
         b.iter(move || {
@@ -56,14 +60,17 @@ pub fn deutsch_jozsa(c: &mut Criterion) {
 pub fn large_file(c: &mut Criterion) {
     c.bench_function("Large file parity evaluation", |b| {
         let sources = SourceMap::new([("large.qs".into(), LARGE.into())], None);
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
+
         b.iter(move || {
             let mut out = Vec::new();
             let mut rec = GenericReceiver::new(&mut out);
@@ -87,14 +94,17 @@ pub fn array_append(c: &mut Criterion) {
                 .into(),
             ),
         );
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
+
         b.iter(move || {
             let mut out = Vec::new();
             let mut rec = GenericReceiver::new(&mut out);
@@ -118,14 +128,17 @@ pub fn array_update(c: &mut Criterion) {
                 .into(),
             ),
         );
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
+
         b.iter(move || {
             let mut out = Vec::new();
             let mut rec = GenericReceiver::new(&mut out);
@@ -137,14 +150,17 @@ pub fn array_update(c: &mut Criterion) {
 pub fn array_literal(c: &mut Criterion) {
     c.bench_function("Array literal evaluation", |b| {
         let sources = SourceMap::new([("none".into(), "".into())], Some(ARRAY_LITERAL.into()));
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
+
         b.iter(move || {
             let mut out = Vec::new();
             let mut rec = GenericReceiver::new(&mut out);
@@ -173,14 +189,17 @@ pub fn large_nested_iteration(c: &mut Criterion) {
                 .into(),
             ),
         );
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let mut evaluator = Interpreter::new(
-            true,
             sources,
             PackageType::Exe,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("code should compile");
+
         b.iter(move || {
             let mut out = Vec::new();
             let mut rec = GenericReceiver::new(&mut out);

compiler/qsc/benches/large.rs

@@ -22,7 +22,7 @@ pub fn large_file(c: &mut Criterion) {
             let sources = SourceMap::new([("large.qs".into(), INPUT.into())], None);
             let (_, reports) = compile(
                 &store,
-                &[std],
+                &[(std, None)],
                 sources,
                 PackageType::Exe,
                 TargetCapabilityFlags::all(),
@@ -37,12 +37,16 @@ pub fn large_file_interpreter(c: &mut Criterion) {
     c.bench_function("Large input file compilation (interpreter)", |b| {
         b.iter(|| {
             let sources = SourceMap::new([("large.qs".into(), INPUT.into())], None);
+            let (std_id, store) =
+                qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
+
             let _evaluator = qsc::interpret::Interpreter::new(
-                true,
                 sources,
                 PackageType::Exe,
                 TargetCapabilityFlags::all(),
                 LanguageFeatures::default(),
+                store,
+                &[(std_id, None)],
             )
             .expect("code should compile");
         });

compiler/qsc/benches/rca.rs

@@ -126,12 +126,14 @@ impl CompilationContext {
 
 impl Default for CompilationContext {
     fn default() -> Self {
+        let (std_id, store) = qsc::compile::package_store_with_stdlib(TargetCapabilityFlags::all());
         let compiler = Compiler::new(
-            true,
             SourceMap::default(),
             PackageType::Lib,
             TargetCapabilityFlags::all(),
             LanguageFeatures::default(),
+            store,
+            &[(std_id, None)],
         )
         .expect("should be able to create a new compiler");
         let fir_store = lower_hir_package_store(compiler.package_store());

compiler/qsc/src/bin/qsc.rs

@@ -7,6 +7,7 @@ use clap::{crate_version, ArgGroup, Parser, ValueEnum};
 use log::info;
 use miette::{Context, IntoDiagnostic, Report};
 use qsc::hir::PackageId;
+use qsc::packages::BuildableProgram;
 use qsc::{compile::compile, PassContext};
 use qsc_codegen::qir::fir_to_qir;
 use qsc_data_structures::{language_features::LanguageFeatures, target::TargetCapabilityFlags};
@@ -18,6 +19,7 @@ use qsc_hir::hir::Package;
 use qsc_partial_eval::ProgramEntry;
 use qsc_passes::PackageType;
 use qsc_project::{FileSystem, StdFs};
+use std::sync::Arc;
 use std::{
     concat, fs,
     io::{self, Read},
@@ -95,64 +97,54 @@ enum Emit {
     Qir,
 }
 
+#[allow(clippy::too_many_lines)]
 fn main() -> miette::Result<ExitCode> {
     env_logger::init();
     let cli = Cli::parse();
-    let mut store = PackageStore::new(qsc::compile::core());
-    let mut dependencies = Vec::new();
     let profile: qsc::target::Profile = cli.profile.unwrap_or_default().into();
     let capabilities = profile.into();
     let package_type = if cli.emit.contains(&Emit::Qir) {
         PackageType::Exe
     } else {
         PackageType::Lib
     };
-
-    if !cli.nostdlib {
-        dependencies.push(store.insert(qsc::compile::std(&store, capabilities)));
-    }
-
     let mut features = LanguageFeatures::from_iter(cli.features);
 
-    let mut sources = cli
-        .sources
-        .iter()
-        .map(read_source)
-        .collect::<miette::Result<Vec<_>>>()?;
-
-    if sources.is_empty() {
-        let fs = StdFs;
-        if let Some(qsharp_json) = cli.qsharp_json {
-            if let Some(dir) = qsharp_json.parent() {
-                let project = match fs.load_project(dir, None) {
-                    Ok(project) => project,
-                    Err(errs) => {
-                        for e in errs {
-                            eprintln!("{e:?}");
-                        }
-                        return Ok(ExitCode::FAILURE);
-                    }
-                };
-
-                let (mut project_sources, language_features) =
-                    project.package_graph_sources.into_sources_temporary();
-
-                sources.append(&mut project_sources);
-
-                features.merge(LanguageFeatures::from_iter(language_features));
-            } else {
-                eprintln!("{} must have a parent directory", qsharp_json.display());
-                return Ok(ExitCode::FAILURE);
+    let (mut store, dependencies, source_map) = if let Some(qsharp_json) = cli.qsharp_json {
+        if let Some(dir) = qsharp_json.parent() {
+            match load_project(dir, &mut features) {
+                Ok(items) => items,
+                Err(exit_code) => return Ok(exit_code),
             }
+        } else {
+            eprintln!("{} must have a parent directory", qsharp_json.display());
+            return Ok(ExitCode::FAILURE);
         }
-    }
+    } else {
+        let sources = cli
+            .sources
+            .iter()
+            .map(read_source)
+            .collect::<miette::Result<Vec<_>>>()?;
+
+        let mut store = PackageStore::new(qsc::compile::core());
+        let dependencies = if cli.nostdlib {
+            vec![]
+        } else {
+            let std_id = store.insert(qsc::compile::std(&store, TargetCapabilityFlags::all()));
+            vec![(std_id, None)]
+        };
+        (
+            store,
+            dependencies,
+            SourceMap::new(sources, cli.entry.clone().map(std::convert::Into::into)),
+        )
+    };
 
-    let entry = cli.entry.unwrap_or_default();
-    let sources = SourceMap::new(sources, Some(entry.into()));
     let (unit, errors) = compile(
         &store,
         &dependencies,
-        sources,
+        source_map,
         package_type,
         capabilities,
         features,
@@ -280,3 +272,54 @@ fn emit_qir(
         }
     }
 }
+
+/// Loads a project from the given directory and returns the package store, the list of
+/// dependencies, and the source map.
+/// Pre-populates the package store with all of the compiled dependencies.
+#[allow(clippy::type_complexity)]
+fn load_project(
+    dir: impl AsRef<Path>,
+    features: &mut LanguageFeatures,
+) -> Result<(PackageStore, Vec<(PackageId, Option<Arc<str>>)>, SourceMap), ExitCode> {
+    let fs = StdFs;
+    let project = match fs.load_project(dir.as_ref(), None) {
+        Ok(project) => project,
+        Err(errs) => {
+            for e in errs {
+                eprintln!("{e:?}");
+            }
+            return Err(ExitCode::FAILURE);
+        }
+    };
+
+    if !project.errors.is_empty() {
+        for e in project.errors {
+            eprintln!("{e:?}");
+        }
+        return Err(ExitCode::FAILURE);
+    }
+
+    // This builds all the dependencies
+    let buildable_program =
+        BuildableProgram::new(TargetCapabilityFlags::all(), project.package_graph_sources);
+
+    if !buildable_program.dependency_errors.is_empty() {
+        for e in buildable_program.dependency_errors {
+            eprintln!("{e:?}");
+        }
+        return Err(ExitCode::FAILURE);
+    }
+
+    let BuildableProgram {
+        store,
+        user_code,
+        user_code_dependencies,
+        ..
+    } = buildable_program;
+
+    let source_map = qsc::SourceMap::new(user_code.sources, None);
+
+    features.merge(LanguageFeatures::from_iter(user_code.language_features));
+
+    Ok((store, user_code_dependencies, source_map))
+}