feat: improve readmeSize, badges and linters in mono-repos

Author: satazor

lib/analyze/collect/metadata.js

@@ -322,8 +322,7 @@ function metadata(data, packageJson) {
             hasSelectiveFiles: Array.isArray(packageJson.files) && packageJson.files.length > 0 ? true : null,
 
             // Need to use typeof because there's some old packages in which the README is an object, e.g.: `flatsite`
-            readme: (typeof data.readme === 'string' && data.readme.indexOf('No README data') === -1) ?
-                data.readme : null,
+            readme: (typeof data.readme === 'string' && data.readme) ? data.readme : null,
         });
     })
     .tap(() => log.debug(`The metadata collector for ${packageJson.name} completed successfully`));

lib/analyze/collect/source.js

@@ -7,14 +7,17 @@ const detectReadmeBadges = require('detect-readme-badges');
 const detectRepoChangelog = require('detect-repo-changelog');
 const fetchCoverage = require('fetch-coverage');
 const loadJsonFile = require('load-json-file');
-const readFile = Promise.promisify(require('fs').readFile);
 const deepCompact = require('deep-compact');
 const isRegularFile = require('is-regular-file');
 const got = require('got');
 const fileSize = require('./util/fileSize');
 const promisePropsSettled = require('./util/promisePropsSettled');
 const exec = require('../util/exec');
 const gotRetry = require('../util/gotRetry');
+const fileContents = require('./util/fileContents');
+const uniqWith = require('lodash/uniqWith');
+const isEqual = require('lodash/isEqual');
+const { uniq } = require('lodash');
 
 const davidBin = path.normalize(`${__dirname}/bin/david-json`);
 const log = logger.child({ module: 'collect/source' });
@@ -28,8 +31,11 @@ const log = logger.child({ module: 'collect/source' });
  * @returns {Promise} The promise for the inspection result.
  */
 function inspectFiles(data, downloaded) {
-    // Readme must be located in the package dir
-    const readmeSize = data.readmeFilename ? fileSize(`${downloaded.packageDir}/${data.readmeFilename}`) : 0;
+    // Sum readme file sizes of the one in the package dir with the root
+    const readmeSize = fileSize([
+        `${downloaded.packageDir}/${data.readmeFilename || 'README.md'}`,
+        downloaded.dir !== downloaded.packageDir ? `${downloaded.DIR}/${data.readmeFilename || 'README.md'}` : '',
+    ].filter((path) => path));
     // Prefer tests located in the package dir and fallback to the root
     const testsSize = (
         detectRepoTestFiles(downloaded.packageDir)
@@ -64,20 +70,17 @@ function inspectFiles(data, downloaded) {
  * @returns {Promise} The promise for the badges result.
  */
 function getReadmeBadges(data, downloaded) {
-    // Prefer README badges from the package dir but usually badges are at the root README
-    // Need to use typeof because there's some old packages in which the README is an object, e.g.: `flatsite`
-    return Promise.try(() => typeof data.readme === 'string' ? detectReadmeBadges(data.readme) : [])
-    .then((badges) => {
-        if (!badges.length && downloaded.dir !== downloaded.packageDir && data.readmeFilename) {
-            return readFile(`${downloaded.dir}/${data.readmeFilename}`)
-            .then((readme) => detectReadmeBadges(readme.toString()))
-            // Ignore if file does not exist or is actually a directory
-            .catch({ code: 'ENOENT' }, () => [])
-            .catch({ code: 'EISDIR' }, () => []);
-        }
-
-        return badges;
-    });
+    // Use badges from both the package dir and root README
+    return Promise.props({
+        onPackageDir: typeof data.readme === 'string' && data.readme ? data.readme : fileContents(`${downloaded.packageDir}/${data.readmeFilename || 'README.md'}`),
+        onRoot: downloaded.dir !== downloaded.packageDir ? fileContents(`${downloaded.dir}/${data.readmeFilename || 'README.md'}`) : '',
+    })
+    .then((readmes) => Promise.props({
+        onRoot: detectReadmeBadges(readmes.onRoot || ''),
+        onPackageDir: detectReadmeBadges(readmes.onPackageDir || ''),
+    }))
+    // Cleanup duplicates
+    .then((badges) => uniqWith([...badges.onPackageDir, ...badges.onRoot], isEqual));
 }
 
 /**
@@ -88,23 +91,22 @@ function getReadmeBadges(data, downloaded) {
  * @returns {Promise} The promise for the linters result.
  */
 function getRepoLinters(downloaded) {
-    // Linters usually are at the root but prefer the ones within the package just in case..
-    return detectRepoLinters(downloaded.packageDir)
-    .then((linters) => {
-        if (linters.length || downloaded.dir === downloaded.packageDir) {
-            return linters;
-        }
-
-        return detectRepoLinters(downloaded.dir)
-        // A JSON error might occur if `detect-repo-linters`fails to parse `package.json` as JSON
-        // Since the `package.json` at the root was not validated, it can have errors
-        // If that's the case, we want to skip them here
-        .catch({ name: 'JSONError' }, () => {
-            log.warn({ dir: downloaded.dir }, 'Error reading downloaded package.json when scanning for linters');
+    // Linters usually are at the root but we consider both just in case..
+    return Promise.props({
+        onPackageDir: detectRepoLinters(downloaded.packageDir),
+        onRootDir: downloaded.dir !== downloaded.packageDir ?
+            detectRepoLinters(downloaded.dir)
+            // A JSON error might occur if `detect-repo-linters`fails to parse `package.json` as JSON
+            // Since the `package.json` at the root was not validated, it can have errors
+            // If that's the case, we want to skip them here
+            .catch({ name: 'JSONError' }, () => {
+                log.warn({ dir: downloaded.dir }, 'Error reading downloaded package.json when scanning for linters');
 
-            return [];
-        });
-    });
+                return [];
+            }) :
+            [],
+    })
+    .then((linters) => uniq([...linters.onPackageDir, ...linters.onRootDir]));
 }
 
 /**

lib/analyze/collect/util/fileContents.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const readFile = Promise.promisify(require('fs').readFile);
+
+const log = logger.child({ module: 'util/file-contents' });
+
+/**
+ * Gets the file contents of a file.
+ *
+ * @param {String} path - The path.
+ *
+ * @returns {Promise} A promise that fulfills when done.
+ */
+function fileContents(path) {
+    return readFile(path)
+    .then((buffer) => buffer.toString())
+    // Return 0 if path does not exist
+    .catch({ code: 'ENOENT' }, () => null)
+    // Return 0 if path is directory
+    .catch({ code: 'EISDIR' }, () => null)
+    // Return 0 if too many symlinks are being followed, e.g.: `condensation`
+    .catch({ code: 'ELOOP' }, (err) => {
+        log.warn({ err }, `ELOOP while getting file size of ${path}, returning 0..`);
+
+        return null;
+    })
+    // Ignore errors of packages that have large nested paths.. e.g.: `cordova-plugin-forcetouch`
+    .catch({ code: 'ENAMETOOLONG' }, (err) => {
+        /* istanbul ignore next */
+        log.warn({ err }, `ENAMETOOLONG while getting file size of ${path}, returning 0..`);
+        /* istanbul ignore next */
+
+        return null;
+    });
+}
+
+module.exports = fileContents;

lib/analyze/evaluate/maintenance.js

@@ -183,7 +183,7 @@ function isPackageFinished(collected) {
     const isStable = semver.gte(collected.metadata.version, '1.0.0', true); // `true` = loose semver
     const isNotDeprecated = !collected.metadata.deprecated;
     const hasFewIssues = get(collected, 'github.issues.openCount', Infinity) < 15;
-    const hasREADME = !!collected.metadata.readme;
+    const hasREADME = !!collected.metadata.readme || get(collected, 'source.files.readmeSize', 0) > 0;
     const hasTests = !!collected.metadata.hasTestScript;
 
     const isFinished = isStable && isNotDeprecated && hasFewIssues && hasREADME && hasTests;