Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(azure-adapter): split transactions by chunks
- Loading branch information
1 parent
3f7d455
commit c0d1761
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| { | ||
| "type": "patch", | ||
| "comment": "feat(azure-adapter): split transactions by chunks", | ||
| "packageName": "monosize-storage-azure", | ||
| "email": "[email protected]", | ||
| "dependentChangeType": "patch" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import type { BundleSizeReport } from 'monosize'; | ||
|
|
||
| export const sampleReport: BundleSizeReport = [ | ||
| { | ||
| packageName: '@scope/foo-package', | ||
| name: 'Foo', | ||
| path: 'foo.fixture.js', | ||
| minifiedSize: 1000, | ||
| gzippedSize: 100, | ||
| }, | ||
| { | ||
| packageName: '@scope/bar-package', | ||
| name: 'Bar', | ||
| path: 'bar.fixture.js', | ||
| minifiedSize: 1000, | ||
| gzippedSize: 100, | ||
| }, | ||
| ]; | ||
|
|
||
| export const bigReport: BundleSizeReport = new Array(200).fill(null).map((_, index) => ({ | ||
| packageName: '@scope/foo-package', | ||
| name: `Entry [${index}]`, | ||
| path: `foo-${index}.fixture.js`, | ||
| minifiedSize: 1000, | ||
| gzippedSize: 100, | ||
| })); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import fetch from 'node-fetch'; | ||
| import pc from 'picocolors'; | ||
| import type { BundleSizeReportEntry, StorageAdapter } from 'monosize'; | ||
|
|
||
| import type { AzureStorageConfig } from './types.mjs'; | ||
|
|
||
| const MAX_HTTP_ATTEMPT_COUNT = 5; | ||
|
|
||
| export function createGetRemoteReport(config: AzureStorageConfig) { | ||
| async function getRemoteReport(branch: string, attempt = 1): ReturnType<StorageAdapter['getRemoteReport']> { | ||
| try { | ||
| const response = await fetch(`${config.endpoint}?branch=${branch}`); | ||
| const result = (await response.json()) as Array<BundleSizeReportEntry & { commitSHA: string }>; | ||
|
|
||
| const remoteReport = result.map(entity => { | ||
| const { commitSHA, ...rest } = entity; | ||
| return rest; | ||
| }); | ||
| const { commitSHA } = result[result.length - 1]; | ||
|
|
||
| return { commitSHA, remoteReport }; | ||
| } catch (err) { | ||
| console.log([pc.yellow('[w]'), (err as Error).toString()].join(' ')); | ||
| console.log([pc.yellow('[w]'), 'Failed to fetch report from the remote. Retrying...'].join(' ')); | ||
|
|
||
| if (attempt >= MAX_HTTP_ATTEMPT_COUNT) { | ||
| console.error( | ||
| [pc.red('[e]'), 'Exceeded 5 attempts to fetch reports, please check previously reported warnings...'].join( | ||
| ' ', | ||
| ), | ||
| ); | ||
| throw err; | ||
| } | ||
|
|
||
| return getRemoteReport(branch, attempt + 1); | ||
| } | ||
| } | ||
|
|
||
| return getRemoteReport; | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| export type AzureStorageConfig = { | ||
| endpoint: string; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| import { AzureNamedKeyCredential, odata, TableClient, TableTransaction } from '@azure/data-tables'; | ||
| import { BundleSizeReportEntry, StorageAdapter } from 'monosize'; | ||
| import pc from 'picocolors'; | ||
|
|
||
| export const ENTRIES_PER_CHUNK = 90; | ||
|
|
||
| export function createRowKey(entry: BundleSizeReportEntry): string { | ||
| // Azure does not support slashes in "rowKey" | ||
| // https://docs.microsoft.com/archive/blogs/jmstall/azure-storage-naming-rules | ||
| return `${entry.packageName}${entry.path.replace(/\.fixture\.js$/, '').replace(/\//g, '')}`; | ||
| } | ||
|
|
||
| export function splitArrayToChunks<T>(arr: T[], size: number): T[][] { | ||
| return [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(i * size, (i + 1) * size)); | ||
| } | ||
|
|
||
| export const uploadReportToRemote: StorageAdapter['uploadReportToRemote'] = async (branch, commitSHA, localReport) => { | ||
| if (typeof process.env['BUNDLESIZE_ACCOUNT_KEY'] !== 'string') { | ||
| throw new Error('monosize-storage-azure: "BUNDLESIZE_ACCOUNT_KEY" is not defined in your process.env'); | ||
| } | ||
|
|
||
| if (typeof process.env['BUNDLESIZE_ACCOUNT_NAME'] !== 'string') { | ||
| throw new Error('monosize-storage-azure: "BUNDLESIZE_ACCOUNT_NAME" is not defined in your process.env'); | ||
| } | ||
|
|
||
| if (localReport.length === 0) { | ||
| console.log([pc.yellow('[w]'), 'No entries to upload'].join(' ')); | ||
| return; | ||
| } | ||
|
|
||
| const AZURE_STORAGE_ACCOUNT = process.env['BUNDLESIZE_ACCOUNT_NAME']; | ||
| const AZURE_STORAGE_TABLE_NAME = 'latest'; | ||
| const AZURE_ACCOUNT_KEY = process.env['BUNDLESIZE_ACCOUNT_KEY']; | ||
|
|
||
| const credentials = new AzureNamedKeyCredential(AZURE_STORAGE_ACCOUNT, AZURE_ACCOUNT_KEY); | ||
| const client = new TableClient( | ||
| `https://${AZURE_STORAGE_ACCOUNT}.table.core.windows.net`, | ||
| AZURE_STORAGE_TABLE_NAME, | ||
| credentials, | ||
| ); | ||
|
|
||
| const transaction = new TableTransaction(); | ||
| const entitiesIterator = await client.listEntities({ | ||
| queryOptions: { | ||
| filter: odata`PartitionKey eq ${branch}`, | ||
| }, | ||
| }); | ||
|
|
||
| for await (const entity of entitiesIterator) { | ||
| // We can't delete and create entries with the same "rowKey" in the same transaction | ||
| // => we delete only entries not present in existing report | ||
| const isEntryPresentInExistingReport = Boolean(localReport.find(entry => createRowKey(entry) === entity.rowKey)); | ||
| const shouldEntryBeDeleted = !isEntryPresentInExistingReport; | ||
|
|
||
| if (shouldEntryBeDeleted) { | ||
| transaction.deleteEntity(entity.partitionKey as string, entity.rowKey as string); | ||
| } | ||
| } | ||
|
|
||
| localReport.forEach(entry => { | ||
| transaction.upsertEntity( | ||
| { | ||
| partitionKey: branch, | ||
| rowKey: createRowKey(entry), | ||
|
|
||
| name: entry.name, | ||
| packageName: entry.packageName, | ||
| path: entry.path, | ||
|
|
||
| minifiedSize: entry.minifiedSize, | ||
| gzippedSize: entry.gzippedSize, | ||
|
|
||
| commitSHA, | ||
| }, | ||
| 'Replace', | ||
| ); | ||
| }); | ||
|
|
||
| const chunks = splitArrayToChunks(transaction.actions, ENTRIES_PER_CHUNK); | ||
|
|
||
| for (const chunk of chunks) { | ||
| await client.submitTransaction(chunk); | ||
| } | ||
| }; |
Oops, something went wrong.