GitHub stats

Author: danwos

.gitignore

@@ -132,6 +132,7 @@ dmypy.json
 
 pypi_data.json
 awesome.json
+pypi_gh_data.json
 .envrc
 
 secrets/*

docs/conf.py

@@ -124,6 +124,7 @@
     'featured',
     'color',
     'deprecated',
+    'stars',
     ]
 
 

docs/index.rst

@@ -54,8 +54,8 @@ Use the export buttons to get the current view as a PDF or Excel file.
 The export takes filter and sorting into account.
 
 .. needtable::
-   :columns: id, title as "Name", content as "Description", sphinx_type as "Type", license, points, monthly as "Monthly downloads", release_days as "Days since last release"
-   :colwidths: 10, 20, 35, 10, 5, 5, 5, 10
+   :columns: id, title as "Name", content as "Description", sphinx_type as "Type", license, points, monthly as "Monthly downloads", release_days as "Days since last release", stars as "Stars"
+   :colwidths: 10, 20, 35,10, 5, 5, 5, 10, 5
    :style_row: awesome_[[copy('color')]]
 
 

scripts/awesome_config.py

@@ -42,11 +42,17 @@
 ##################################################
 # NEEDS_JSON.PY configs
 ##################################################
-#PYPI_FILE = 'pypi_data.json'
-PYPI_FILE = 'data/20221024_pypi_data.json'
+PYPI_FILE = 'pypi_gh_data.json'
+#PYPI_FILE = 'data/20221024_pypi_data.json'
 
 NEED_FILE = 'awesome.json'
 
 NEED_TYPE = 'sw'
 
 MAX_NEEDS = 5000  # User for  development for faster tests
+
+
+##################################################
+# GITHUB_STATS.PY configs
+##################################################
+GH_JSON_FILE = 'pypi_gh_data.json'

scripts/github_stats.py

@@ -0,0 +1,102 @@
+"""
+Collects GitHub stats from project in pypi_data.json
+"""
+import json
+import sys
+import os
+from threading import Thread
+from github import Github, UnknownObjectException
+
+# Make Python aware of the awesom_config file
+sys.path.append(os.path.dirname(__file__))
+from awesome_config import *
+
+
+KEY = "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"
+
+
+def get_gh_topics(name, gh_project, results, counter):
+    awesome_tags = []
+    awesome_stars = -1
+    try:
+        repo = g.get_repo(gh_project)
+        awesome_tags = repo.get_topics()
+        awesome_stars = repo.stargazers_count
+        print(f'{counter}/{len(pypi_data)} {name}: Tags retrieved: {awesome_tags}')
+    except UnknownObjectException:
+        pass
+
+    results[name] = {
+        'tags': awesome_tags,
+        'stars': awesome_stars
+    }
+
+
+
+print(f'Reading pypi data from {PYPI_FILE}')
+with open(PYPI_FILE, 'r') as f:
+    pypi_data = json.load(f)
+
+gh_projects={}
+
+for name, project  in pypi_data.items():
+    # Collect possible github repository URLS
+    from contextlib import suppress
+
+    gh_urls = []
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['project_urls']['Github'])
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['project_urls']['Code'])
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['project_urls']['Source Code'])
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['project_urls']['Source'])
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['project_urls']['Homepage'])
+    with suppress(KeyError,TypeError):
+        gh_urls.append(project['info']['home_page'])
+
+        
+    gh_urls_clean = []
+    for url in gh_urls:
+        if url is not None and 'github.com' in url and '/issues' not in url:
+            url_clean = url.replace('https://github.com/', '').strip('/')
+            gh_urls_clean.append(url_clean)
+
+    if gh_urls_clean:
+        gh_projects[name] = gh_urls_clean[0]
+        
+print(f'Github projects found: {len(gh_projects)}/{len(pypi_data)}')
+
+g = Github(KEY)
+counter = 0
+results = {}
+threads = {}
+
+for name, project in pypi_data.items():
+    
+    awesome_tags = []
+    if name in gh_projects:
+        gh_project = gh_projects[name]
+        threads[name] = Thread(target=get_gh_topics, args=(name, gh_project, results, counter))
+        threads[name].start()
+        counter += 1
+
+print(f'Received {len(results)} tag-results from GitHub')
+
+for name, project in pypi_data.items():
+    project['awesome_stats']['tags'] = []
+    project['awesome_stats']['stars'] = -1
+    if name in results:
+        project['awesome_stats']['tags'] = results[name]['tags']
+        project['awesome_stats']['stars'] = results[name]['stars']
+
+print(f'Storing data into {GH_JSON_FILE}')
+with open(GH_JSON_FILE, 'w') as f:
+    json.dump(pypi_data, f, sort_keys=True, indent=4)
+
+print('Done. Exit now!')
+
+# for repo in g.get_user().get_repos():
+#     print(repo.name)

scripts/needs_json.py

@@ -70,6 +70,16 @@ def classifiers_check(value, classifiers):
     except KeyError:
         overall = 0
 
+    try:
+        tags = data['awesome_stats']['tags']
+    except KeyError:
+        tags = []
+
+    try:
+        stars = data['awesome_stats']['stars']
+    except KeyError:
+        stars = 0
+
     # urls
     code = ""
     if data['info']['project_urls']:
@@ -98,8 +108,9 @@ def classifiers_check(value, classifiers):
         "sphinx_type": sphinx_type,
         "license": license,
         "monthly": monthly,
-        "overall": overall,
-        "tags": [],
+        # "overall": overall,  # currently not collected 
+        "tags": tags,
+        "stars": stars,
         "pypi": data['info']['package_url'],
         "code": code,
         "website": data['info']['home_page'],