Skip to content

Commit 41e6fe4

Browse files
arcanisarcanis
authored
refactor!: move ZipFS and ZipOpenFS into @yarnpkg/libzip (#4853)
* feat: Moves ZipFS and ZipOpenFS into @yarnpkg/libzip ZipFS and ZipOpenFS were until now stored alongside the rest of the filesystem utilities. While it made some sense to keep all the FakeFS implementation together, it started to become a little awkward after most of the ZipOpenFS logic was moved inside a generic MountFS, leaving ZipFS / ZipOpenFS more related to an external dependency than they were to the package that contained them. It also made it difficult to build tools leveraging them (such as the `mountMemoryDrive` function mentioned below), because we always had to dance with the `libzip` parameter when constructing instances. This diff changes that, moving ZipFS and ZipOpenFS into `@yarnpkg/libzip`. The benefit API-wise is that since ZipFS and ZipOpenFS are now store alongside libzip, the libzip instance no longer has to be specified when constructing ZipFS / ZipOpenFS instances (i.e. you can now run `new ZipFS()` instead of `new ZipFS(null, {libzip})`). It should work on both Node.js and browser environments, although on browsers it'll require that the ZipFS / ZipOpenFS instances are only constructed after `getLibzipPromise` returned. On Node.js, the libzip will be lazily loaded. This diff also exposes a new utility called `mountMemoryDrive`. It's an helper whose purpose is to transparently mount memory drives on disk. While not directly related to zip itself, it uses an in-memory zip instance to store the disk mutations. It would (perhaps?) be more optimized to use a dedicated in-memory FakeFS instance, but since it would require to reimplement a lot of the logic that ZipFS already does I don't foresee doing this job anytime soon, if at all. In the meantime, ZipFS is a fine place to store this tool. * Version * Fixes unit tests * Fixes constraint * Fixes the lockfile
1 parent f9609dd commit 41e6fe4

35 files changed

+534
-456
lines changed

‎.pnp.cjs

+84-64
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.yarn/versions/5be37cc3.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
releases:
2+
"@yarnpkg/cli": major
3+
"@yarnpkg/core": major
4+
"@yarnpkg/fslib": major
5+
"@yarnpkg/libzip": major
6+
7+
declined:
8+
- "@yarnpkg/plugin-compat"
9+
- "@yarnpkg/plugin-constraints"
10+
- "@yarnpkg/plugin-dlx"
11+
- "@yarnpkg/plugin-essentials"
12+
- "@yarnpkg/plugin-exec"
13+
- "@yarnpkg/plugin-file"
14+
- "@yarnpkg/plugin-git"
15+
- "@yarnpkg/plugin-github"
16+
- "@yarnpkg/plugin-http"
17+
- "@yarnpkg/plugin-init"
18+
- "@yarnpkg/plugin-interactive-tools"
19+
- "@yarnpkg/plugin-link"
20+
- "@yarnpkg/plugin-nm"
21+
- "@yarnpkg/plugin-npm"
22+
- "@yarnpkg/plugin-npm-cli"
23+
- "@yarnpkg/plugin-pack"
24+
- "@yarnpkg/plugin-patch"
25+
- "@yarnpkg/plugin-pnp"
26+
- "@yarnpkg/plugin-pnpm"
27+
- "@yarnpkg/plugin-stage"
28+
- "@yarnpkg/plugin-typescript"
29+
- "@yarnpkg/plugin-version"
30+
- "@yarnpkg/plugin-workspace-tools"
31+
- vscode-zipfs
32+
- "@yarnpkg/builder"
33+
- "@yarnpkg/doctor"
34+
- "@yarnpkg/extensions"
35+
- "@yarnpkg/nm"
36+
- "@yarnpkg/pnp"
37+
- "@yarnpkg/pnpify"
38+
- "@yarnpkg/sdks"
39+
- "@yarnpkg/shell"

‎packages/plugin-file/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
},
1010
"dependencies": {
1111
"@yarnpkg/fslib": "workspace:^",
12+
"@yarnpkg/libzip": "workspace:^",
1213
"tslib": "^2.4.0"
1314
},
1415
"peerDependencies": {

‎packages/plugin-file/sources/fileUtils.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {structUtils, FetchOptions, Locator, miscUtils, tgzUtils, Ident, FetchResult} from '@yarnpkg/core';
2-
import {ppath, PortablePath, npath, CwdFS, ZipFS} from '@yarnpkg/fslib';
2+
import {ppath, PortablePath, npath, CwdFS} from '@yarnpkg/fslib';
3+
import {ZipFS} from '@yarnpkg/libzip';
34

45
export function parseSpec(spec: string) {
56
const {params, selector} = structUtils.parseRange(spec);

‎packages/plugin-nm/sources/NodeModulesLinker.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import {Linker, LinkOptions, MinimalLinkOptions, LinkType} from
44
import {LocatorHash, Descriptor, DependencyMeta, Configuration} from '@yarnpkg/core';
55
import {MessageName, Project, FetchResult, Installer} from '@yarnpkg/core';
66
import {PortablePath, npath, ppath, toFilename, Filename} from '@yarnpkg/fslib';
7-
import {VirtualFS, ZipOpenFS, xfs, FakeFS, NativePath} from '@yarnpkg/fslib';
8-
import {getLibzipPromise} from '@yarnpkg/libzip';
7+
import {VirtualFS, xfs, FakeFS, NativePath} from '@yarnpkg/fslib';
8+
import {ZipOpenFS} from '@yarnpkg/libzip';
99
import {buildNodeModulesTree} from '@yarnpkg/nm';
1010
import {NodeModulesLocatorMap, buildLocatorMap, NodeModulesHoistingLimits} from '@yarnpkg/nm';
1111
import {parseSyml} from '@yarnpkg/parsers';
@@ -219,7 +219,6 @@ class NodeModulesInstaller implements Installer {
219219

220220
const defaultFsLayer = new VirtualFS({
221221
baseFs: new ZipOpenFS({
222-
libzip: await getLibzipPromise(),
223222
maxOpenFiles: 80,
224223
readOnlyArchives: true,
225224
}),

‎packages/plugin-nm/sources/PnpLooseLinker.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {LinkOptions, structUtils} from '@yarnpkg/core';
2-
import {VirtualFS, ZipOpenFS, ppath, Filename} from '@yarnpkg/fslib';
3-
import {getLibzipPromise} from '@yarnpkg/libzip';
2+
import {VirtualFS, ppath, Filename} from '@yarnpkg/fslib';
3+
import {ZipOpenFS} from '@yarnpkg/libzip';
44
import {NodeModulesPackageNode, buildNodeModulesTree} from '@yarnpkg/nm';
55
import {PnpInstaller, PnpLinker} from '@yarnpkg/plugin-pnp';
66
import {PnpSettings, makeRuntimeApi, DependencyTarget} from '@yarnpkg/pnp';
@@ -19,7 +19,6 @@ class PnpLooseInstaller extends PnpInstaller {
1919
async transformPnpSettings(pnpSettings: PnpSettings) {
2020
const defaultFsLayer = new VirtualFS({
2121
baseFs: new ZipOpenFS({
22-
libzip: await getLibzipPromise(),
2322
maxOpenFiles: 80,
2423
readOnlyArchives: true,
2524
}),

‎packages/plugin-patch/sources/PatchFetcher.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import {Fetcher, FetchOptions, MinimalFetchOptions, ReportError, MessageName, Report} from '@yarnpkg/core';
22
import {Locator} from '@yarnpkg/core';
33
import {miscUtils, structUtils} from '@yarnpkg/core';
4-
import {ppath, xfs, ZipFS, Filename, CwdFS, PortablePath} from '@yarnpkg/fslib';
5-
import {getLibzipPromise} from '@yarnpkg/libzip';
4+
import {ppath, xfs, Filename, CwdFS, PortablePath} from '@yarnpkg/fslib';
5+
import {ZipFS} from '@yarnpkg/libzip';
66

77
import * as patchUtils from './patchUtils';
88
import {UnmatchedHunkError} from './tools/UnmatchedHunkError';
@@ -49,11 +49,8 @@ export class PatchFetcher implements Fetcher {
4949
const sourceFetch = await opts.fetcher.fetch(sourceLocator, opts);
5050
const prefixPath = structUtils.getIdentVendorPath(locator);
5151

52-
const libzip = await getLibzipPromise();
53-
5452
// First we create a copy of the package that we'll be free to mutate
5553
const initialCopy = new ZipFS(currentFile, {
56-
libzip,
5754
create: true,
5855
level: opts.project.configuration.get(`compressionLevel`),
5956
});
@@ -73,7 +70,6 @@ export class PatchFetcher implements Fetcher {
7370
// single time) because it lets us easily rollback when hitting errors
7471
// on optional patches (we just need to call `discardAndClose`).
7572
const patchedPackage = new ZipFS(currentFile, {
76-
libzip,
7773
level: opts.project.configuration.get(`compressionLevel`),
7874
});
7975

@@ -122,7 +118,6 @@ export class PatchFetcher implements Fetcher {
122118
}
123119

124120
return new ZipFS(currentFile, {
125-
libzip,
126121
level: opts.project.configuration.get(`compressionLevel`),
127122
});
128123
}

‎packages/vscode-zipfs/sources/ZipFSProvider.ts

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
1-
import {ZipOpenFS, VirtualFS, PosixFS, npath} from '@yarnpkg/fslib';
2-
import {getLibzipSync} from '@yarnpkg/libzip';
3-
import * as vscode from 'vscode';
1+
import {VirtualFS, PosixFS, npath} from '@yarnpkg/fslib';
2+
import {ZipOpenFS} from '@yarnpkg/libzip';
3+
import * as vscode from 'vscode';
44

55
export class ZipFSProvider implements vscode.FileSystemProvider {
66
private readonly fs = new PosixFS(
77
new VirtualFS({
88
baseFs: new ZipOpenFS({
9-
libzip: getLibzipSync(),
109
useCache: true,
1110
maxOpenFiles: 80,
1211
}),

‎packages/yarnpkg-core/sources/Cache.ts

+15-17
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
import {FakeFS, LazyFS, NodeFS, ZipFS, PortablePath, Filename, AliasFS} from '@yarnpkg/fslib';
2-
import {ppath, xfs, DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/fslib';
3-
import {getLibzipPromise} from '@yarnpkg/libzip';
4-
import {randomBytes} from 'crypto';
5-
import fs from 'fs';
6-
7-
import {Configuration} from './Configuration';
8-
import {MessageName} from './MessageName';
9-
import {ReportError} from './Report';
10-
import * as hashUtils from './hashUtils';
11-
import * as miscUtils from './miscUtils';
12-
import * as structUtils from './structUtils';
13-
import {LocatorHash, Locator} from './types';
1+
import {FakeFS, LazyFS, NodeFS, PortablePath, Filename, AliasFS} from '@yarnpkg/fslib';
2+
import {ppath, xfs} from '@yarnpkg/fslib';
3+
import {DEFAULT_COMPRESSION_LEVEL, ZipFS} from '@yarnpkg/libzip';
4+
import {randomBytes} from 'crypto';
5+
import fs from 'fs';
6+
7+
import {Configuration} from './Configuration';
8+
import {MessageName} from './MessageName';
9+
import {ReportError} from './Report';
10+
import * as hashUtils from './hashUtils';
11+
import * as miscUtils from './miscUtils';
12+
import * as structUtils from './structUtils';
13+
import {LocatorHash, Locator} from './types';
1414

1515
const CACHE_VERSION = 9;
1616

@@ -172,7 +172,7 @@ export class Cache {
172172
// it seem like it actually exist on the disk, at the location of the
173173
// cache the package would fill if it was normally fetched.
174174
const makeMockPackage = () => {
175-
const zipFs = new ZipFS(null, {libzip});
175+
const zipFs = new ZipFS();
176176

177177
const rootPackageDir = ppath.join(PortablePath.root, structUtils.getIdentVendorPath(locator));
178178
zipFs.mkdirSync(rootPackageDir, {recursive: true});
@@ -363,11 +363,9 @@ export class Cache {
363363

364364
let zipFs: ZipFS | undefined;
365365

366-
const libzip = await getLibzipPromise();
367-
368366
const zipFsBuilder = shouldMock
369367
? () => makeMockPackage()
370-
: () => new ZipFS(cachePath, {baseFs, libzip, readOnly: true});
368+
: () => new ZipFS(cachePath, {baseFs, readOnly: true});
371369

372370
const lazyFs = new LazyFS<PortablePath>(() => miscUtils.prettifySyncErrors(() => {
373371
return zipFs = zipFsBuilder();

‎packages/yarnpkg-core/sources/Configuration.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import {Filename, PortablePath, npath, ppath, xfs} from '@yarnpkg/fslib';
2-
import {DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/fslib';
2+
import {DEFAULT_COMPRESSION_LEVEL} from '@yarnpkg/libzip';
33
import {parseSyml, stringifySyml} from '@yarnpkg/parsers';
44
import camelcase from 'camelcase';
55
import {isCI, isPR, GITHUB_ACTIONS} from 'ci-info';

‎packages/yarnpkg-core/sources/scriptUtils.ts

+22-26
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
1-
import {CwdFS, Filename, NativePath, PortablePath, ZipOpenFS} from '@yarnpkg/fslib';
2-
import {xfs, npath, ppath, toFilename} from '@yarnpkg/fslib';
3-
import {getLibzipPromise} from '@yarnpkg/libzip';
4-
import {execute} from '@yarnpkg/shell';
5-
import capitalize from 'lodash/capitalize';
6-
import pLimit from 'p-limit';
7-
import {PassThrough, Readable, Writable} from 'stream';
8-
9-
import {Configuration} from './Configuration';
10-
import {Manifest} from './Manifest';
11-
import {MessageName} from './MessageName';
12-
import {Project} from './Project';
13-
import {ReportError, Report} from './Report';
14-
import {StreamReport} from './StreamReport';
15-
import {Workspace} from './Workspace';
16-
import {YarnVersion} from './YarnVersion';
17-
import * as execUtils from './execUtils';
18-
import * as formatUtils from './formatUtils';
19-
import * as miscUtils from './miscUtils';
20-
import * as semverUtils from './semverUtils';
21-
import * as structUtils from './structUtils';
22-
import {LocatorHash, Locator} from './types';
1+
import {CwdFS, Filename, NativePath, PortablePath} from '@yarnpkg/fslib';
2+
import {xfs, npath, ppath, toFilename} from '@yarnpkg/fslib';
3+
import {ZipOpenFS} from '@yarnpkg/libzip';
4+
import {execute} from '@yarnpkg/shell';
5+
import capitalize from 'lodash/capitalize';
6+
import pLimit from 'p-limit';
7+
import {PassThrough, Readable, Writable} from 'stream';
8+
9+
import {Configuration} from './Configuration';
10+
import {Manifest} from './Manifest';
11+
import {MessageName} from './MessageName';
12+
import {Project} from './Project';
13+
import {ReportError, Report} from './Report';
14+
import {StreamReport} from './StreamReport';
15+
import {Workspace} from './Workspace';
16+
import {YarnVersion} from './YarnVersion';
17+
import * as execUtils from './execUtils';
18+
import * as formatUtils from './formatUtils';
19+
import * as miscUtils from './miscUtils';
20+
import * as semverUtils from './semverUtils';
21+
import * as structUtils from './structUtils';
22+
import {LocatorHash, Locator} from './types';
2323

2424
/**
2525
* @internal
@@ -432,8 +432,6 @@ export async function hasPackageScript(locator: Locator, scriptName: string, {pr
432432
const manifest = await Manifest.find(PortablePath.dot, {baseFs: packageFs});
433433

434434
return manifest.scripts.has(scriptName);
435-
}, {
436-
libzip: await getLibzipPromise(),
437435
});
438436
}
439437

@@ -545,8 +543,6 @@ async function initializePackageEnvironment(locator: Locator, {project, binFolde
545543
cwd = packageLocation;
546544

547545
return {manifest, binFolder, env, cwd};
548-
}, {
549-
libzip: await getLibzipPromise(),
550546
});
551547
}
552548

‎packages/yarnpkg-core/sources/tgzUtils.ts

+10-12
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import {Filename, FakeFS, PortablePath, ZipCompression, ZipFS, NodeFS, ppath, xfs, npath, constants} from '@yarnpkg/fslib';
2-
import {getLibzipPromise} from '@yarnpkg/libzip';
3-
import {PassThrough, Readable} from 'stream';
4-
import tar from 'tar';
1+
import {Filename, FakeFS, PortablePath, NodeFS, ppath, xfs, npath, constants} from '@yarnpkg/fslib';
2+
import {ZipCompression, ZipFS} from '@yarnpkg/libzip';
3+
import {PassThrough, Readable} from 'stream';
4+
import tar from 'tar';
55

6-
import {WorkerPool} from './WorkerPool';
7-
import * as miscUtils from './miscUtils';
8-
import {getContent as getZipWorkerSource, ConvertToZipPayload} from './worker-zip';
6+
import {WorkerPool} from './WorkerPool';
7+
import * as miscUtils from './miscUtils';
8+
import {getContent as getZipWorkerSource, ConvertToZipPayload} from './worker-zip';
99

1010
interface MakeArchiveFromDirectoryOptions {
1111
baseFs?: FakeFS<PortablePath>;
@@ -15,16 +15,14 @@ interface MakeArchiveFromDirectoryOptions {
1515
}
1616

1717
export async function makeArchiveFromDirectory(source: PortablePath, {baseFs = new NodeFS(), prefixPath = PortablePath.root, compressionLevel, inMemory = false}: MakeArchiveFromDirectoryOptions = {}): Promise<ZipFS> {
18-
const libzip = await getLibzipPromise();
19-
2018
let zipFs;
2119
if (inMemory) {
22-
zipFs = new ZipFS(null, {libzip, level: compressionLevel});
20+
zipFs = new ZipFS(null, {level: compressionLevel});
2321
} else {
2422
const tmpFolder = await xfs.mktempPromise();
2523
const tmpFile = ppath.join(tmpFolder, `archive.zip` as Filename);
2624

27-
zipFs = new ZipFS(tmpFile, {create: true, libzip, level: compressionLevel});
25+
zipFs = new ZipFS(tmpFile, {create: true, level: compressionLevel});
2826
}
2927

3028
const target = ppath.resolve(PortablePath.root, prefixPath!);
@@ -49,7 +47,7 @@ export async function convertToZip(tgz: Buffer, opts: ExtractBufferOptions) {
4947

5048
await workerPool.run({tmpFile, tgz, opts});
5149

52-
return new ZipFS(tmpFile, {libzip: await getLibzipPromise(), level: opts.compressionLevel});
50+
return new ZipFS(tmpFile, {level: opts.compressionLevel});
5351
}
5452

5553
async function * parseTar(tgz: Buffer) {

‎packages/yarnpkg-core/sources/worker-zip/Worker.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import {PortablePath, statUtils, ZipFS} from '@yarnpkg/fslib';
2-
import {getLibzipPromise} from '@yarnpkg/libzip';
1+
import {PortablePath, statUtils} from '@yarnpkg/fslib';
2+
import {ZipFS} from '@yarnpkg/libzip';
33
import {parentPort} from 'worker_threads';
44

55
import {extractArchiveTo, ExtractBufferOptions} from '../tgzUtils';
@@ -13,7 +13,7 @@ parentPort.on(`message`, async (data: ConvertToZipPayload) => {
1313
const {opts, tgz, tmpFile} = data;
1414
const {compressionLevel, ...bufferOpts} = opts;
1515

16-
const zipFs = new ZipFS(tmpFile, {create: true, libzip: await getLibzipPromise(), level: compressionLevel, stats: statUtils.makeDefaultStats()});
16+
const zipFs = new ZipFS(tmpFile, {create: true, level: compressionLevel, stats: statUtils.makeDefaultStats()});
1717

1818
// Buffers sent through Node are turned into regular Uint8Arrays
1919
const tgzBuffer = Buffer.from(tgz.buffer, tgz.byteOffset, tgz.byteLength);

‎packages/yarnpkg-core/tests/TestPlugin.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {Descriptor, Fetcher, FetchOptions, LinkType, Locator, MinimalResolveOptions, Package, Plugin, ResolveOptions, Resolver, structUtils} from '@yarnpkg/core';
2-
import {PortablePath, xfs, ZipFS} from '@yarnpkg/fslib';
3-
import {getLibzipPromise} from '@yarnpkg/libzip';
2+
import {PortablePath, xfs} from '@yarnpkg/fslib';
3+
import {ZipFS} from '@yarnpkg/libzip';
44

55
export class UnboundDescriptorResolver implements Resolver {
66
supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) {
@@ -148,7 +148,7 @@ class NoopFetcher implements Fetcher {
148148

149149
async fetch(locator: Locator, opts: FetchOptions) {
150150
const tempDir = await xfs.mktempPromise();
151-
return {packageFs: new ZipFS(`${tempDir}/archive.zip` as PortablePath, {libzip: await getLibzipPromise(), create: true}), prefixPath: PortablePath.dot};
151+
return {packageFs: new ZipFS(`${tempDir}/archive.zip` as PortablePath, {create: true}), prefixPath: PortablePath.dot};
152152
}
153153
}
154154

0 commit comments

Comments
 (0)