Introduce Q# project structure (#794)

# Q# Projects

This PR introduces a Q# manifest file, `qsharp.json`, and a
corresponding notion of _Q# projects_.

## Feature Overview
When the compiler is invoked, it looks upward in the directory structure
for a manifest file. Upon finding a valid manifest, we set that
directory as the root directory of the project. We then include all
`*.qs` files in that directory or subdirectories of
that directory as sources, unless they are explicitly excluded in the
manifest. This implementation *does not follow symlinks of any kind*.

All FS requirements are expressed as the `FileSystem` interface. This
will provide compatibility when we integrate projects with the language
service and other virtual file systems. `qsc_project` only utilizes
`std::fs` if the `fs` feature is enabled.

## API Changes

There are some notable differences in the behavior of `qsc` and `qsi`:
* `qsc` still accepts sources as input, but now, if no sources are
specified, `qsc` will look for a directory in the current directory.
* Currently, `qsc` requires a list of source names on the command line.
With this PR, if `qsc` is invoked within a project with
  no sources specified, it will include project sources.
* `qsi` will include sources within its project scope if it is invoked
within a project. A flag `--no-init-project`/`-n`
   has been added to skip this.

## Testing
`qsc_project` is tested using our normal `expect_test` workflow. There
are a series of test projects in `qsc_project/tests` that represent
various project structures and manifests. We expect certain directory
structures and manifests to deterministically correspond to a set of
source names and source contents, and this is the interface that is
tested.

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "compiler/qsc_hir",
     "compiler/qsc_parse",
     "compiler/qsc_passes",
+    "compiler/qsc_project",
     "fuzz",
     "katas",
     "language_service",

compiler/qsc/Cargo.toml

@@ -24,6 +24,7 @@ qsc_ast = { path = "../qsc_ast" }
 qsc_fir = { path = "../qsc_fir" }
 qsc_hir = { path = "../qsc_hir" }
 qsc_passes = { path = "../qsc_passes" }
+qsc_project = { path = "../qsc_project", features = ["fs"] }
 thiserror = { workspace = true }
 
 [dev-dependencies]

compiler/qsc/src/bin/qsc.rs

@@ -15,6 +15,7 @@ use qsc_frontend::{
 };
 use qsc_hir::hir::{Package, PackageId};
 use qsc_passes::PackageType;
+use qsc_project::{FileSystem, Manifest, StdFs};
 use std::{
     concat, fs,
     io::{self, Read},
@@ -24,8 +25,8 @@ use std::{
 };
 
 #[derive(Debug, Parser)]
-#[command(version = concat!(crate_version!(), " (", env!("QSHARP_GIT_HASH"), ")"), arg_required_else_help(true))]
-#[clap(group(ArgGroup::new("input").args(["entry", "sources"]).required(true).multiple(true)))]
+#[command(version = concat!(crate_version!(), " (", env!("QSHARP_GIT_HASH"), ")"), arg_required_else_help(false))]
+#[clap(group(ArgGroup::new("input").args(["entry", "sources"]).required(false).multiple(true)))]
 struct Cli {
     /// Disable automatic inclusion of the standard library.
     #[arg(long)]
@@ -74,12 +75,23 @@ fn main() -> miette::Result<ExitCode> {
         dependencies.push(store.insert(qsc::compile::std(&store, target)));
     }
 
-    let sources = cli
+    let mut sources = cli
         .sources
         .iter()
         .map(read_source)
         .collect::<miette::Result<Vec<_>>>()?;
 
+    if sources.is_empty() {
+        let fs = StdFs;
+        let manifest = Manifest::load()?;
+        if let Some(manifest) = manifest {
+            let project = fs.load_project(manifest)?;
+            let mut project_sources = project.sources;
+
+            sources.append(&mut project_sources);
+        }
+    }
+
     let entry = cli.entry.unwrap_or_default();
     let sources = SourceMap::new(sources, Some(entry.into()));
     let (unit, errors) = compile(&store, &dependencies, sources, package_type, target);

compiler/qsc/src/bin/qsi.rs

@@ -18,6 +18,7 @@ use qsc_eval::{
 };
 use qsc_frontend::compile::{SourceContents, SourceMap, SourceName};
 use qsc_passes::PackageType;
+use qsc_project::{FileSystem, Manifest, StdFs};
 use std::{
     fs,
     io::{self, prelude::BufRead, Write},
@@ -72,12 +73,22 @@ impl Receiver for TerminalReceiver {
 
 fn main() -> miette::Result<ExitCode> {
     let cli = Cli::parse();
-    let sources = cli
+    let mut sources = cli
         .sources
         .iter()
         .map(read_source)
         .collect::<miette::Result<Vec<_>>>()?;
 
+    if sources.is_empty() {
+        let fs = StdFs;
+        let manifest = Manifest::load()?;
+        if let Some(manifest) = manifest {
+            let project = fs.load_project(manifest)?;
+            let mut project_sources = project.sources;
+
+            sources.append(&mut project_sources);
+        }
+    }
     if cli.exec {
         let mut interpreter = match Interpreter::new(
             !cli.nostdlib,

compiler/qsc_project/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "qsc_project"
+
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+edition.workspace = true
+license.workspace = true
+version.workspace = true
+
+
+[dependencies]
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+miette = { workspace = true }
+regex-lite = { workspace = true }
+
+[dev-dependencies]
+expect-test = { workspace = true }
+qsc_project = { path = ".", features = ["fs"] }
+
+[features]
+fs = []

compiler/qsc_project/src/error.rs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use miette::Diagnostic;
+use thiserror::Error;
+
+#[derive(Error, Debug, Diagnostic)]
+pub enum Error {
+    #[error("found a qsharp.json file, but it was invalid: {0}")]
+    SerdeJson(#[from] serde_json::Error),
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    #[error("failed to construct regular expression from excluded file item: {0}")]
+    RegexError(#[from] regex_lite::Error),
+}

compiler/qsc_project/src/fs.rs

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! This module contains a project implementation using [std::fs].
+
+use crate::{DirEntry, EntryType, FileSystem};
+use miette::{Context, IntoDiagnostic};
+use std::fs::DirEntry as StdEntry;
+use std::path::Path;
+use std::{path::PathBuf, sync::Arc};
+
+/// This struct represents management of Q# projects from the [std::fs] filesystem implementation.
+#[derive(Default)]
+pub struct StdFs;
+
+impl DirEntry for StdEntry {
+    type Error = crate::Error;
+    fn entry_type(&self) -> Result<EntryType, Self::Error> {
+        Ok(self.file_type()?.into())
+    }
+
+    fn extension(&self) -> String {
+        self.path()
+            .extension()
+            .map(|x| x.to_string_lossy().to_string())
+            .unwrap_or_default()
+    }
+
+    fn entry_name(&self) -> String {
+        self.file_name().to_string_lossy().to_string()
+    }
+
+    fn path(&self) -> PathBuf {
+        self.path()
+    }
+}
+
+impl std::convert::From<std::fs::FileType> for EntryType {
+    fn from(file_type: std::fs::FileType) -> Self {
+        if file_type.is_dir() {
+            EntryType::Folder
+        } else if file_type.is_file() {
+            EntryType::File
+        } else if file_type.is_symlink() {
+            EntryType::Symlink
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+impl FileSystem for StdFs {
+    type Entry = StdEntry;
+
+    fn read_file(&self, path: &Path) -> miette::Result<(Arc<str>, Arc<str>)> {
+        let contents = std::fs::read_to_string(path)
+            .into_diagnostic()
+            .with_context(|| format!("could not read source file `{}`", path.display()))?;
+
+        Ok((path.to_string_lossy().into(), contents.into()))
+    }
+
+    fn list_directory(&self, path: &Path) -> miette::Result<Vec<StdEntry>> {
+        let listing = std::fs::read_dir(path).map_err(crate::Error::from)?;
+        Ok(listing
+            .collect::<Result<_, _>>()
+            .map_err(crate::Error::from)?)
+    }
+}

compiler/qsc_project/src/lib.rs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! This module handles the logic that constitutes the Q# project system.
+//! This includes locating a manifest file in the filesystem, loading and parsing
+//! the manifest, and determining which files are members of the project.
+
+mod error;
+#[cfg(feature = "fs")]
+mod fs;
+mod manifest;
+mod project;
+
+pub use error::Error;
+#[cfg(feature = "fs")]
+pub use fs::StdFs;
+pub use manifest::{Manifest, ManifestDescriptor, MANIFEST_FILE_NAME};
+pub use project::{DirEntry, EntryType, FileSystem, Project};

compiler/qsc_project/src/manifest.rs

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#[cfg(feature = "fs")]
+use crate::Error;
+#[cfg(feature = "fs")]
+use std::{
+    env::current_dir,
+    fs::{self, DirEntry, FileType},
+};
+
+use regex_lite::Regex;
+use serde::Deserialize;
+use std::path::PathBuf;
+
+pub const MANIFEST_FILE_NAME: &str = "qsharp.json";
+
+/// A Q# manifest, used to describe project metadata.
+#[derive(Deserialize, Debug, Default)]
+pub struct Manifest {
+    pub author: Option<String>,
+    pub license: Option<String>,
+    #[serde(default)]
+    pub exclude_regexes: Vec<String>,
+    #[serde(default)]
+    pub exclude_files: Vec<String>,
+}
+
+/// Describes the contents and location of a Q# manifest file.
+#[derive(Debug)]
+pub struct ManifestDescriptor {
+    pub manifest: Manifest,
+    pub manifest_dir: PathBuf,
+}
+
+impl ManifestDescriptor {
+    pub(crate) fn exclude_regexes(&self) -> Result<Vec<Regex>, crate::Error> {
+        self.manifest
+            .exclude_regexes
+            .iter()
+            .map(|x| Regex::new(x))
+            .collect::<Result<_, _>>()
+            .map_err(crate::Error::from)
+    }
+
+    pub(crate) fn exclude_files(&self) -> &[String] {
+        &self.manifest.exclude_files
+    }
+}
+
+#[cfg(feature = "fs")]
+impl Manifest {
+    /// Starting from the current directory, traverse ancestors until
+    /// a manifest is found.
+    /// Returns an error if there are any filesystem errors, or if
+    /// a manifest file exists but is the wrong format.
+    /// Returns `Ok(None)` if there is no file matching the manifest file
+    /// name.
+    pub fn load() -> std::result::Result<Option<ManifestDescriptor>, Error> {
+        let current_dir = current_dir()?;
+        Self::load_from_path(current_dir)
+    }
+
+    /// Given a [PathBuf], traverse [PathBuf::ancestors] until a Manifest is found.
+    /// Returns [None] if no manifest named [MANIFEST_FILE_NAME] is found.
+    /// Returns an error if a manifest is found, but is not parsable into the
+    /// expected format.
+    pub fn load_from_path(path: PathBuf) -> std::result::Result<Option<ManifestDescriptor>, Error> {
+        let ancestors = path.ancestors();
+        for ancestor in ancestors {
+            let listing = ancestor.read_dir()?;
+            for item in listing.into_iter().filter_map(only_valid_files) {
+                if item.file_name().to_str() == Some(MANIFEST_FILE_NAME) {
+                    let mut manifest_dir = item.path();
+                    // pop off the file name itself
+                    manifest_dir.pop();
+
+                    let manifest = fs::read_to_string(item.path())?;
+                    let manifest = serde_json::from_str(&manifest)?;
+                    return Ok(Some(ManifestDescriptor {
+                        manifest,
+                        manifest_dir,
+                    }));
+                }
+            }
+        }
+        Ok(None)
+    }
+}
+
+/// Utility function which filters out any [DirEntry] which is not a valid file or
+/// was unable to be read.
+#[cfg(feature = "fs")]
+fn only_valid_files(item: std::result::Result<DirEntry, std::io::Error>) -> Option<DirEntry> {
+    match item {
+        Ok(item)
+            if (item
+                .file_type()
+                .as_ref()
+                .map(FileType::is_file)
+                .unwrap_or_default()) =>
+        {
+            Some(item)
+        }
+        _ => None,
+    }
+}