Fix packaging for WHL format (pytorch#656)

Author: joecummings

MANIFEST.in

@@ -1,9 +1 @@
-# Add necessary recipe files (not as importable)
-recursive-include recipes *.py *.yaml
-
-# Add requirements
-include requirements.txt
-include dev-requirements.txt
-
-# Remove tests from packaging
-prune tests
+prune tests  # Remove all testing files from final dist/

dev-requirements.txt

@@ -1,3 +1,69 @@
+# ---- All project specifications ---- #
+[project]
+name = "torchtune"
+description = "A native-PyTorch library for LLM fine-tuning"
+readme = "README.md"
+requires-python = ">=3.8"
+license = {file = "LICENSE"}
+authors = [
+    { name = "PyTorch Team", email = "packages@pytorch.org" },
+]
+keywords = ["pytorch", "finetuning", "llm"]
+dependencies = [
+    # Hugging Face integrations
+    "datasets",
+    "huggingface_hub",
+    "safetensors",
+
+    # Miscellaneous
+    "sentencepiece",
+    "tqdm",
+    "omegaconf",
+
+    # Quantization
+    "torchao==0.1",
+]
+dynamic = ["version"]
+
+[project.urls]
+GitHub = "https://github.com/pytorch/torchtune"
+Documentation = "https://pytorch.org/torchtune/main/index.html"
+Issues = "https://github.com/pytorch/torchtune/issues"
+
+[project.scripts]
+tune = "torchtune._cli.tune:main"
+
+[project.optional-dependencies]
+dev = [
+    "bitsandbytes",
+    "pre-commit",
+    "pytest",
+    "pytest-cov",
+    "pytest-mock",
+    "pytest-integration",
+    "tensorboard",
+    "transformers",
+    "wandb",
+]
+
+[tool.setuptools.dynamic]
+version = {attr = "torchtune.__version__"}
+
+
+# ---- Explicit project build information ---- #
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = [""]
+include = ["torchtune*", "recipes*"]
+
+[tool.setuptools.package-data]
+recipes = ["configs/*.yaml", "configs/*/*.yaml"]
+
+
+# ---- Tooling specifications ---- #
 [tool.usort]
 first_party_detection = false
 

pyproject.toml

@@ -18,7 +18,6 @@
 
 # TODO: Add proper link to pytorch.org/torchtune/... when the docs are live.
 raise ModuleNotFoundError(
-    "The TorchTune recipes folder isn't a package and you should not import anything from there. "
-    "Refer to our docs for detailed instructions on how to use the recipes!"
-    "This file only exists for testing reasons."
+    "The TorchTune recipes directory isn't a package and you should not import anything from here. "
+    "Refer to our docs for detailed instructions on how to use recipes!"
 )

recipes/__init__.py

@@ -6,10 +6,12 @@
 import os
 from pathlib import Path
 
+import torchtune
+
 from omegaconf import OmegaConf
 from torchtune import config
 
-CONFIG_DIR = Path(__file__).resolve().parent.parent.parent / "recipes" / "configs"
+CONFIG_DIR = Path(torchtune.__file__).parent.parent / "recipes" / "configs"
 
 
 class TestConfigs:

requirements.txt

@@ -8,5 +8,7 @@
 
 
 def test_import_recipes():
-    with pytest.raises(ModuleNotFoundError, match="recipes folder isn't a package"):
+    with pytest.raises(
+        ModuleNotFoundError, match="The TorchTune recipes directory isn't a package"
+    ):
         import recipes  # noqa

setup.py

@@ -4,6 +4,8 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+__version__ = "0.0.1"
+
 from torchtune import datasets, models, modules, utils
 
 __all__ = [datasets, models, modules, utils]