From 8dc0546a3c63d514e76fb9b355e288c26d058f2f Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Sat, 8 Jul 2023 14:49:28 -0700 Subject: [PATCH] feat: use git remote for branch related config This will check the origin remote if it exists and use that to determine which branches exist. These branches are then used to populate CI branches, branch protections, and dependabot. Using this for dependabot is a new feature which allows old release branches to get dependency updates for template-oss only. This also updates the dependabot config to only update the root directory instead of each workspace directory. The previous way was an attempt to get it to work with workspaces, but wasn't used in any our repos. Dependabot should now be able to update workspaces when configured to use a single root directory. Fixes #329 --- .github/dependabot.yml | 13 +-- .github/settings.yml | 26 ----- .github/workflows/ci-test-workspace.yml | 2 - .github/workflows/ci.yml | 2 - .github/workflows/codeql-analysis.yml | 4 - .github/workflows/release.yml | 2 - lib/config.js | 14 ++- lib/content/_on-ci.yml | 2 +- lib/content/codeql-analysis.yml | 4 +- lib/content/dependabot.yml | 13 ++- lib/content/index.js | 12 +-- lib/content/release.yml | 2 +- lib/util/dependabot.js | 28 ++++++ lib/util/get-git-url.js | 26 ----- lib/util/git.js | 74 ++++++++++++++ .../test/apply/source-snapshots.js.test.cjs | 75 ++++---------- test/apply/dependabot.js | 99 ++++++++++++++++--- test/apply/merge-yml.js | 82 +++------------ test/fixtures/yml-merge.js | 8 ++ test/setup.js | 10 +- 20 files changed, 265 insertions(+), 233 deletions(-) create mode 100644 lib/util/dependabot.js delete mode 100644 lib/util/get-git-url.js create mode 100644 lib/util/git.js diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5b1b6e99..69312dfc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,18 +7,7 @@ updates: directory: / schedule: interval: daily - allow: - - dependency-type: direct - versioning-strategy: increase-if-necessary - commit-message: - prefix: deps - prefix-development: chore - labels: - - "Dependencies" - - package-ecosystem: npm - directory: workspace/test-workspace/ - schedule: - interval: daily + target-branch: "main" allow: - dependency-type: direct versioning-strategy: increase-if-necessary diff --git a/.github/settings.yml b/.github/settings.yml index adbef7e6..107aa0ad 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -24,29 +24,3 @@ branches: apps: [] users: [] teams: [ "cli-team" ] - - name: latest - protection: - required_status_checks: null - enforce_admins: true - required_pull_request_reviews: - required_approving_review_count: 1 - require_code_owner_reviews: true - require_last_push_approval: true - dismiss_stale_reviews: true - restrictions: - apps: [] - users: [] - teams: [ "cli-team" ] - - name: release/v* - protection: - required_status_checks: null - enforce_admins: true - required_pull_request_reviews: - required_approving_review_count: 1 - require_code_owner_reviews: true - require_last_push_approval: true - dismiss_stale_reviews: true - restrictions: - apps: [] - users: [] - teams: [ "cli-team" ] diff --git a/.github/workflows/ci-test-workspace.yml b/.github/workflows/ci-test-workspace.yml index 5e88d011..80794556 100644 --- a/.github/workflows/ci-test-workspace.yml +++ b/.github/workflows/ci-test-workspace.yml @@ -10,8 +10,6 @@ on: push: branches: - main - - latest - - release/v* paths: - workspace/test-workspace/** schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00c8a131..7802dc43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,6 @@ on: push: branches: - main - - latest - - release/v* paths-ignore: - workspace/test-workspace/** schedule: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 21244879..f7e691d9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -6,13 +6,9 @@ on: push: branches: - main - - latest - - release/v* pull_request: branches: - main - - latest - - release/v* schedule: # "At 10:00 UTC (03:00 PT) on Monday" https://crontab.guru/#0_10_*_*_1 - cron: "0 10 * * 1" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 339dd69e..ae19cd6f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ on: push: branches: - main - - latest - - release/v* permissions: contents: write diff --git a/lib/config.js b/lib/config.js index d5acc35a..748ffe32 100644 --- a/lib/config.js +++ b/lib/config.js @@ -2,7 +2,8 @@ const { relative, dirname, join, extname, posix, win32 } = require('path') const { defaults, pick, omit, uniq } = require('lodash') const semver = require('semver') const parseCIVersions = require('./util/parse-ci-versions.js') -const getGitUrl = require('./util/get-git-url.js') +const parseDependabot = require('./util/dependabot.js') +const git = require('./util/git.js') const gitignore = require('./util/gitignore.js') const { mergeWithArrays } = require('./util/merge.js') const { FILE_KEYS, parseConfig: parseFiles, getAddedFiles, mergeFiles } = require('./util/files.js') @@ -153,6 +154,9 @@ const getFullConfig = async ({ const publicPkgs = pkgs.filter(p => !p.pkgJson.private) const allPrivate = pkgs.every(p => p.pkgJson.private) + const branches = uniq([...pkgConfig.branches ?? [], pkgConfig.releaseBranch]).filter(Boolean) + const gitBranches = await git.getBranches(rootPkg.path, branches) + // all derived keys const derived = { isRoot, @@ -170,6 +174,12 @@ const getFullConfig = async ({ allPrivate, // controls whether we are in a monorepo with any public workspaces isMonoPublic: isMono && !!publicPkgs.filter(p => p.path !== rootPkg.path).length, + // git + defaultBranch: await git.defaultBranch(rootPkg.path), + branches: gitBranches.branches, + branchPatterns: gitBranches.patterns, + // dependabot + dependabot: parseDependabot(pkgConfig, defaultConfig, gitBranches.branches), // repo repoDir: rootPkg.path, repoFiles, @@ -261,7 +271,7 @@ const getFullConfig = async ({ } } - const gitUrl = await getGitUrl(rootPkg.path) + const gitUrl = await git.getUrl(rootPkg.path) if (gitUrl) { derived.repository = { type: 'git', diff --git a/lib/content/_on-ci.yml b/lib/content/_on-ci.yml index 151b31ba..5395447a 100644 --- a/lib/content/_on-ci.yml +++ b/lib/content/_on-ci.yml @@ -12,7 +12,7 @@ pull_request: {{/if}} push: branches: - {{#each branches}} + {{#each branchPatterns}} - {{ . }} {{/each}} {{#if isWorkspace}} diff --git a/lib/content/codeql-analysis.yml b/lib/content/codeql-analysis.yml index 4e4c18f4..4903a0be 100644 --- a/lib/content/codeql-analysis.yml +++ b/lib/content/codeql-analysis.yml @@ -3,12 +3,12 @@ name: CodeQL on: push: branches: - {{#each branches}} + {{#each branchPatterns}} - {{ . }} {{/each}} pull_request: branches: - {{#each branches}} + {{#each branchPatterns}} - {{ . }} {{/each}} schedule: diff --git a/lib/content/dependabot.yml b/lib/content/dependabot.yml index 0f747f9d..fb2d5e2f 100644 --- a/lib/content/dependabot.yml +++ b/lib/content/dependabot.yml @@ -1,15 +1,24 @@ version: 2 updates: + {{#each dependabot}} - package-ecosystem: npm - directory: {{ pkgDir }} + directory: / schedule: interval: daily + target-branch: "{{ branch }}" allow: - dependency-type: direct - versioning-strategy: {{ dependabot }} + {{#each allowNames }} + dependency-name: "{{ . }}" + {{/each}} + versioning-strategy: {{ strategy }} commit-message: prefix: deps prefix-development: chore labels: - "Dependencies" + {{#each labels }} + - "{{ . }}" + {{/each}} + {{/each}} diff --git a/lib/content/index.js b/lib/content/index.js index 185ed46f..0e7f6ba5 100644 --- a/lib/content/index.js +++ b/lib/content/index.js @@ -38,14 +38,6 @@ const sharedRootAdd = (name) => ({ '.github/dependabot.yml': { file: 'dependabot.yml', filter: (p) => p.config.dependabot, - clean: (p) => p.config.isRoot, - // dependabot takes a single top level config file. this parser - // will run for all configured packages and each one will have - // its item replaced in the updates array based on the directory - parser: (p) => class extends p.YmlMerge { - key = 'updates' - id = 'directory' - }, }, '.github/workflows/post-dependabot.yml': { file: 'post-dependabot.yml', @@ -139,8 +131,8 @@ module.exports = { workspaceModule, windowsCI: true, macCI: true, - branches: ['main', 'latest', 'release/v*'], - defaultBranch: 'main', + branches: ['main', 'latest'], + releaseBranch: 'release/v*', distPaths: [ 'bin/', 'lib/', diff --git a/lib/content/release.yml b/lib/content/release.yml index 9f5f5f97..976e1e90 100644 --- a/lib/content/release.yml +++ b/lib/content/release.yml @@ -8,7 +8,7 @@ on: type: string push: branches: - {{#each branches}} + {{#each branchPatterns}} - {{ . }} {{/each}} diff --git a/lib/util/dependabot.js b/lib/util/dependabot.js new file mode 100644 index 00000000..376f90dd --- /dev/null +++ b/lib/util/dependabot.js @@ -0,0 +1,28 @@ +const { name: NAME } = require('../../package.json') +const { minimatch } = require('minimatch') + +const parseDependabotConfig = (v) => typeof v === 'string' ? { strategy: v } : (v ?? {}) + +module.exports = (config, defaultConfig, branches) => { + const { dependabot } = config + const { dependabot: defaultDependabot } = defaultConfig + + if (!dependabot) { + return false + } + + return branches + .filter((b) => dependabot[b] !== false) + .map(branch => { + const isRelease = minimatch(branch, config.releaseBranch) + + return { + branch, + allowNames: isRelease ? [NAME] : [], + labels: isRelease ? ['Backport', branch] : [], + ...parseDependabotConfig(defaultDependabot), + ...parseDependabotConfig(dependabot), + ...parseDependabotConfig(dependabot[branch]), + } + }) +} diff --git a/lib/util/get-git-url.js b/lib/util/get-git-url.js deleted file mode 100644 index 94715b88..00000000 --- a/lib/util/get-git-url.js +++ /dev/null @@ -1,26 +0,0 @@ -const hgi = require('hosted-git-info') -const git = require('@npmcli/git') - -// parse a repo from a git origin into a format -// for a package.json#repository object -const getRepo = async (path) => { - if (!await git.is({ cwd: path })) { - return - } - - try { - const res = await git.spawn([ - 'remote', - 'get-url', - 'origin', - ], { cwd: path }) - const { domain, user, project } = hgi.fromUrl(res.stdout.trim()) - const url = new URL(`https://${domain}`) - url.pathname = `/${user}/${project}.git` - return url.toString() - } catch { - // errors are ignored - } -} - -module.exports = getRepo diff --git a/lib/util/git.js b/lib/util/git.js new file mode 100644 index 00000000..ae821db8 --- /dev/null +++ b/lib/util/git.js @@ -0,0 +1,74 @@ +const hgi = require('hosted-git-info') +const git = require('@npmcli/git') +const { minimatch } = require('minimatch') + +const cache = new Map() + +const tryGit = async (path, ...args) => { + if (!await git.is({ cwd: path })) { + throw new Error('no git') + } + const key = [path, ...args].join(',') + if (cache.has(key)) { + return cache.get(key) + } + const res = git.spawn(args, { cwd: path }).then(r => r.stdout.trim()) + cache.set(key, res) + return res +} + +// parse a repo from a git origin into a format +// for a package.json#repository object +const getUrl = async (path) => { + try { + const urlStr = await tryGit(path, 'remote', 'get-url', 'origin') + const { domain, user, project } = hgi.fromUrl(urlStr) + const url = new URL(`https://${domain}`) + url.pathname = `/${user}/${project}.git` + return url.toString() + } catch { + // errors are ignored + } +} + +const getBranches = async (path, branchPatterns) => { + let matchingBranches = new Set() + let matchingPatterns = new Set() + + try { + const res = await tryGit(path, 'ls-remote', '--heads', 'origin').then(r => r.split('\n')) + const remotes = res.map((h) => h.match(/refs\/heads\/(.*)$/)).filter(Boolean).map(h => h[1]) + for (const branch of remotes) { + for (const pattern of branchPatterns) { + if (minimatch(branch, pattern)) { + matchingBranches.add(branch) + matchingPatterns.add(pattern) + } + } + } + } catch { + matchingBranches = new Set(branchPatterns.filter(b => !b.includes('*'))) + matchingPatterns = new Set(branchPatterns) + } + + return { + branches: [...matchingBranches], + patterns: [...matchingPatterns], + } +} + +const defaultBranch = async (path) => { + try { + const remotes = await tryGit(path, 'remote', 'show', 'origin') + const branch = remotes.match(/HEAD branch: (.*)$/m) + return branch[1] + } catch { + return 'main' + } +} + +module.exports = { + getUrl, + getBranches, + defaultBranch, +} diff --git a/tap-snapshots/test/apply/source-snapshots.js.test.cjs b/tap-snapshots/test/apply/source-snapshots.js.test.cjs index c20c3258..5891dda6 100644 --- a/tap-snapshots/test/apply/source-snapshots.js.test.cjs +++ b/tap-snapshots/test/apply/source-snapshots.js.test.cjs @@ -56,6 +56,20 @@ updates: directory: / schedule: interval: daily + target-branch: "main" + allow: + - dependency-type: direct + versioning-strategy: increase-if-necessary + commit-message: + prefix: deps + prefix-development: chore + labels: + - "Dependencies" + - package-ecosystem: npm + directory: / + schedule: + interval: daily + target-branch: "latest" allow: - dependency-type: direct versioning-strategy: increase-if-necessary @@ -204,19 +218,6 @@ branches: apps: [] users: [] teams: [ "cli-team" ] - - name: release/v* - protection: - required_status_checks: null - enforce_admins: true - required_pull_request_reviews: - required_approving_review_count: 1 - require_code_owner_reviews: true - require_last_push_approval: true - dismiss_stale_reviews: true - restrictions: - apps: [] - users: [] - teams: [ "cli-team" ] .github/workflows/audit.yml ======================================== @@ -1497,6 +1498,7 @@ updates: directory: / schedule: interval: daily + target-branch: "main" allow: - dependency-type: direct versioning-strategy: increase-if-necessary @@ -1506,21 +1508,10 @@ updates: labels: - "Dependencies" - package-ecosystem: npm - directory: workspaces/a/ - schedule: - interval: daily - allow: - - dependency-type: direct - versioning-strategy: increase-if-necessary - commit-message: - prefix: deps - prefix-development: chore - labels: - - "Dependencies" - - package-ecosystem: npm - directory: workspaces/b/ + directory: / schedule: interval: daily + target-branch: "latest" allow: - dependency-type: direct versioning-strategy: increase-if-necessary @@ -1669,19 +1660,6 @@ branches: apps: [] users: [] teams: [ "cli-team" ] - - name: release/v* - protection: - required_status_checks: null - enforce_admins: true - required_pull_request_reviews: - required_approving_review_count: 1 - require_code_owner_reviews: true - require_last_push_approval: true - dismiss_stale_reviews: true - restrictions: - apps: [] - users: [] - teams: [ "cli-team" ] .github/workflows/audit.yml ======================================== @@ -3312,9 +3290,10 @@ version: 2 updates: - package-ecosystem: npm - directory: workspaces/a/ + directory: / schedule: interval: daily + target-branch: "main" allow: - dependency-type: direct versioning-strategy: increase-if-necessary @@ -3324,9 +3303,10 @@ updates: labels: - "Dependencies" - package-ecosystem: npm - directory: workspaces/b/ + directory: / schedule: interval: daily + target-branch: "latest" allow: - dependency-type: direct versioning-strategy: increase-if-necessary @@ -3412,19 +3392,6 @@ branches: apps: [] users: [] teams: [ "cli-team" ] - - name: release/v* - protection: - required_status_checks: null - enforce_admins: true - required_pull_request_reviews: - required_approving_review_count: 1 - require_code_owner_reviews: true - require_last_push_approval: true - dismiss_stale_reviews: true - restrictions: - apps: [] - users: [] - teams: [ "cli-team" ] .github/workflows/ci-a.yml ======================================== diff --git a/test/apply/dependabot.js b/test/apply/dependabot.js index 2a0fdd01..f3c2a33c 100644 --- a/test/apply/dependabot.js +++ b/test/apply/dependabot.js @@ -1,30 +1,97 @@ const t = require('tap') +const yaml = require('yaml') const setup = require('../setup.js') -t.test('default dependabot', async (t) => { - const s = await setup(t) +const setupDependabot = async (t, { branches = ['main'], ...config } = {}) => { + const s = await setup(t, { + package: { + templateOSS: config, + }, + mocks: { + '@npmcli/git': { + is: async () => true, + spawn: async (args) => { + const command = args.filter(a => typeof a === 'string').join(' ') + if (command === 'ls-remote --heads origin') { + return { + stdout: branches.map(b => `xxxxx refs/heads/${b}`).join('\n'), + } + } + }, + }, + }, + }) await s.apply() + const postDependabot = await s.readFile('.github/workflows/post-dependabot.yml') + .catch(() => false) const dependabot = await s.readFile('.github/dependabot.yml') - const postDependabot = await s.stat('.github/workflows/post-dependabot.yml') + .then(r => yaml.parse(r).updates) + .catch(() => false) + + return { + ...s, + dependabot, + postDependabot, + } +} - t.match(dependabot, 'increase-if-necessary') - t.ok(postDependabot) +t.test('default', async (t) => { + const s = await setupDependabot(t) + + t.equal(s.dependabot.length, 1) + t.strictSame(s.dependabot[0], { + 'package-ecosystem': 'npm', + directory: '/', + schedule: { interval: 'daily' }, + 'target-branch': 'main', + allow: [{ 'dependency-type': 'direct' }], + 'versioning-strategy': 'increase-if-necessary', + 'commit-message': { prefix: 'deps', 'prefix-development': 'chore' }, + labels: ['Dependencies'], + }) + + t.ok(s.postDependabot) }) -t.test('no dependabot', async (t) => { - const s = await setup(t, { - package: { - templateOSS: { - dependabot: false, - }, +t.test('change strategy', async (t) => { + const s = await setupDependabot(t, { + dependabot: 'some-other-strategy', + }) + + t.equal(s.dependabot[0]['versioning-strategy'], 'some-other-strategy') +}) + +t.test('turn off specific branch', async (t) => { + const s = await setupDependabot(t, { + dependabot: { + main: false, }, }) - await s.apply() + t.equal(s.dependabot, null) +}) - const dependabot = await s.stat('.github/dependabot.yml').catch(() => false) - const postDependabot = await s.stat('.github/workflows/post-dependabot.yml').catch(() => false) +t.test('release brancheses', async (t) => { + const s = await setupDependabot(t, { + branches: [ + 'release/v10', + ], + }) + + t.match(s.dependabot[0], { + 'target-branch': 'release/v10', + allow: [{ + 'dependency-type': 'direct', + 'dependency-name': '@npmcli/template-oss', + }], + labels: ['Dependencies', 'Backport', 'release/v10'], + }) +}) - t.equal(dependabot, false) - t.equal(postDependabot, false) +t.test('no dependabot', async (t) => { + const s = await setupDependabot(t, { + dependabot: false, + }) + t.equal(s.dependabot, false) + t.equal(s.postDependabot, false) }) diff --git a/test/apply/merge-yml.js b/test/apply/merge-yml.js index 1303353f..d80e50f7 100644 --- a/test/apply/merge-yml.js +++ b/test/apply/merge-yml.js @@ -1,5 +1,4 @@ const t = require('tap') -const { join } = require('path') const yaml = require('yaml') const setup = require('../setup.js') @@ -22,6 +21,14 @@ t.test('json merge', async (t) => { { noid: 1 }, ], }), + 'clean-target.yml': toYml({ + existing: 'header', + key: [ + { id: 1, a: 1 }, + { id: 2, a: 2 }, + { noid: 1 }, + ], + }), content: { 'index.js': await setup.fixture('yml-merge.js'), 'source.yml': toYml({ @@ -47,71 +54,12 @@ t.test('json merge', async (t) => { { id: 3, b: 3 }, ], }) -}) - -t.test('dependabot', async t => { - t.test('root', async (t) => { - const s = await setup(t, { - ok: true, - }) - await s.apply() - - const dependabot = await s.readFile(join('.github', 'dependabot.yml')) - - t.match(dependabot, 'directory: /') - t.notMatch(dependabot, /directory: workspaces/) - - t.same(await s.check(), []) - await s.apply() - await s.apply() - await s.apply() - t.same(await s.check(), []) - }) - - t.test('root + workspaces', async (t) => { - const s = await setup(t, { - ok: true, - workspaces: { a: 'a', b: 'b', c: 'c' }, - }) - await s.apply() - - const dependabot = await s.readFile(join('.github', 'dependabot.yml')) - - t.match(dependabot, 'directory: /') - t.match(dependabot, 'directory: workspaces/a/') - t.match(dependabot, 'directory: workspaces/b/') - t.match(dependabot, 'directory: workspaces/c/') - - t.same(await s.check(), []) - await s.apply() - await s.apply() - await s.apply() - t.same(await s.check(), []) - }) - - t.test('workspaces only', async (t) => { - const s = await setup(t, { - ok: true, - package: { - templateOSS: { - rootRepo: false, - }, - }, - workspaces: { a: 'a', b: 'b', c: 'c' }, - }) - await s.apply() - - const dependabot = await s.readFile(join('.github', 'dependabot.yml')) - - t.notMatch(dependabot, /directory: \//) - t.match(dependabot, 'directory: workspaces/a/') - t.match(dependabot, 'directory: workspaces/b/') - t.match(dependabot, 'directory: workspaces/c/') - - t.same(await s.check(), []) - await s.apply() - await s.apply() - await s.apply() - t.same(await s.check(), []) + t.strictSame(yaml.parse(await s.readFile('clean-target.yml')), { + new: 'header', + key: [ + { id: 1, b: 1 }, + { id: 2, b: 2 }, + { id: 3, b: 3 }, + ], }) }) diff --git a/test/fixtures/yml-merge.js b/test/fixtures/yml-merge.js index 3b9edd9d..26e891bd 100644 --- a/test/fixtures/yml-merge.js +++ b/test/fixtures/yml-merge.js @@ -8,6 +8,14 @@ module.exports = { id = 'id' }, }, + 'clean-target.yml': { + file: 'source.yml', + clean: () => true, + parser: (p) => class extends p.YmlMerge { + key = 'key' + id = 'id' + }, + }, }, }, } diff --git a/test/setup.js b/test/setup.js index 5593ffca..92b52bb5 100644 --- a/test/setup.js +++ b/test/setup.js @@ -6,8 +6,6 @@ const Git = require('@npmcli/git') const localeCompare = require('@isaacs/string-locale-compare')('en') const npa = require('npm-package-arg') const output = require('../lib/util/output.js') -const apply = require('../lib/apply/index.js') -const check = require('../lib/check/index.js') const CONTENT = require('..') const { name: NAME, version: VERSION } = require('../package.json') @@ -43,7 +41,7 @@ const okPackage = () => Object.entries(CONTENT.requiredPackages) }, }) -const setupRoot = async (root) => { +const setupRoot = async (t, root, mocks) => { const rootPath = (...p) => join(root, ...p) // fs methods for reading from the root @@ -83,6 +81,9 @@ const setupRoot = async (root) => { return Object.fromEntries(files.map((f, i) => [f, contents[i]])) } + const apply = t.mock('../lib/apply/index.js', mocks) + const check = t.mock('../lib/check/index.js', mocks) + return { root, ...rootFs, @@ -101,6 +102,7 @@ const setup = async (t, { package = {}, workspaces = {}, testdir = {}, + mocks = {}, ok, } = {}) => { const wsLookup = {} @@ -139,7 +141,7 @@ const setup = async (t, { )) return { - ...(await setupRoot(root)), + ...(await setupRoot(t, root, mocks)), workspaces: wsLookup, } }