Fix case-insensitive set operations

data/character-class-escape-sets.js

@@ -2,6 +2,7 @@
 'use strict';
 
 const regenerate = require('regenerate');
+const UNICODE_IV_SET = require('./all-characters.js').UNICODE_IV_SET;
 
 exports.REGULAR = new Map([
 	['d', regenerate()
@@ -103,3 +104,24 @@ exports.UNICODE_IGNORE_CASE = new Map([
 		.addRange(0x180, 0x2129)
 		.addRange(0x212B, 0x10FFFF)]
 ]);
+
+exports.UNICODESET_IGNORE_CASE = new Map([
+	['d', regenerate()
+		.addRange(0x30, 0x39)],
+	['D', UNICODE_IV_SET.clone().remove(regenerate()
+		.addRange(0x30, 0x39))],
+	['s', regenerate(0x20, 0xA0, 0x1680, 0x202F, 0x205F, 0x3000, 0xFEFF)
+		.addRange(0x9, 0xD)
+		.addRange(0x2000, 0x200A)
+		.addRange(0x2028, 0x2029)],
+	['S', UNICODE_IV_SET.clone().remove(regenerate(0x20, 0xA0, 0x1680, 0x202F, 0x205F, 0x3000, 0xFEFF)
+		.addRange(0x9, 0xD)
+		.addRange(0x2000, 0x200A)
+		.addRange(0x2028, 0x2029))],
+	['w', regenerate(0x5F)
+		.addRange(0x30, 0x39)
+		.addRange(0x61, 0x7A)],
+	['W', UNICODE_IV_SET.clone().remove(regenerate(0x5F)
+		.addRange(0x30, 0x39)
+		.addRange(0x61, 0x7A))]
+]);

package.json

@@ -42,6 +42,7 @@
 		"data/all-characters.js",
 		"data/character-class-escape-sets.js",
 		"data/i-bmp-mappings.js",
+		"data/iu-foldings.js",
 		"data/iu-mappings.js"
 	],
 	"scripts": {

scripts/case-mappings.js

@@ -194,3 +194,4 @@ const iBMPMappings = flattenMapping(filteredBMPMappings);
 
 writeMap('data/i-bmp-mappings.js', iBMPMappings);
 writeMap('data/iu-mappings.js', iuMappings);
+writeMap('data/iu-foldings.js', oneWayMappings);

scripts/character-class-escape-sets.js

@@ -8,21 +8,25 @@ require('./utils/regenerate-plugin-to-code.js');
 const Zs = require('@unicode/unicode-16.0.0/General_Category/Space_Separator/code-points.js');
 
 const iuMappings = require('../data/iu-mappings.js');
+const iuFoldings = require('../data/iu-foldings.js');
+const { UNICODE_SET, UNICODE_IV_SET } = require('../data/all-characters.js');
 
-const caseFold = (codePoint) => {
+const simpleCaseFolding = (codePoint) => {
+	return iuFoldings.get(codePoint) || codePoint;
+};
+
+const getCaseEquivalents = (codePoint) => {
 	return iuMappings.get(codePoint) || false;
 };
 
-// Prepare a Regenerate set containing all code points, used for negative
-// character classes (if any).
-const UNICODE_SET = regenerate().addRange(0x0, 0x10FFFF);
 // Without the `u` flag, the range stops at 0xFFFF.
 // https://mths.be/es#sec-pattern-semantics
 const BMP_SET = regenerate().addRange(0x0, 0xFFFF);
 
 const ESCAPE_CHARS = {};
 const ESCAPE_CHARS_UNICODE = {};
 const ESCAPE_CHARS_UNICODE_IGNORE_CASE = {};
+const ESCAPE_CHARS_UNICODESET_IGNORE_CASE = {};
 const addCharacterClassEscape = (lower, set) => {
 	ESCAPE_CHARS[lower] = ESCAPE_CHARS_UNICODE[lower] = set;
 	const upper = lower.toUpperCase();
@@ -33,24 +37,34 @@ const addCharacterClassEscape = (lower, set) => {
 	// regular expressions that have both the `u` and `i` flags set.
 	const codePoints = set.toArray();
 	const iuSet = regenerate();
-	let containsFoldingSymbols = false;
+	let containsSimpleCaseFolding = false;
 	for (const codePoint of codePoints) {
-		let folded = caseFold(codePoint);
-		if (folded) {
-			containsFoldingSymbols = true;
-			iuSet.add(folded);
-			folded = caseFold(folded);
-			if (folded) {
-				iuSet.add(folded);
+		let caseEquivalents = getCaseEquivalents(codePoint);
+		if (caseEquivalents) {
+			containsSimpleCaseFolding = true;
+			iuSet.add(caseEquivalents);
+			caseEquivalents = getCaseEquivalents(caseEquivalents);
+			if (caseEquivalents) {
+				iuSet.add(caseEquivalents);
 			}
 		}
 	}
-	const iuLowerSet = containsFoldingSymbols ?
+	const iuLowerSet = containsSimpleCaseFolding ?
 		iuSet.clone().add(set) :
 		set;
 	const iuUpperSet = UNICODE_SET.clone().remove(iuLowerSet);
 	ESCAPE_CHARS_UNICODE_IGNORE_CASE[lower] = iuLowerSet;
 	ESCAPE_CHARS_UNICODE_IGNORE_CASE[upper] = iuUpperSet;
+
+	ESCAPE_CHARS_UNICODESET_IGNORE_CASE[lower] = regenerate(
+		iuLowerSet.toArray().map(ch => simpleCaseFolding(ch))
+	);
+
+	ESCAPE_CHARS_UNICODESET_IGNORE_CASE[upper] = {
+		toCode() {
+			return 'UNICODE_IV_SET.clone().remove(' + ESCAPE_CHARS_UNICODESET_IGNORE_CASE[lower].toCode() + ')';
+		}
+	}
 }
 
 // Prepare a Regenerate set for every existing character class escape.
@@ -94,10 +108,11 @@ const stringify = (name, object) => {
 
 const source = [
 	'// Generated using `npm run build`. Do not edit.\n' +
-	`'use strict';\n\nconst regenerate = require('regenerate');`,
+	`'use strict';\n\nconst regenerate = require('regenerate');\nconst UNICODE_IV_SET = require('./all-characters.js').UNICODE_IV_SET`,
 	stringify('REGULAR', ESCAPE_CHARS),
 	stringify('UNICODE', ESCAPE_CHARS_UNICODE),
-	stringify('UNICODE_IGNORE_CASE', ESCAPE_CHARS_UNICODE_IGNORE_CASE)
+	stringify('UNICODE_IGNORE_CASE', ESCAPE_CHARS_UNICODE_IGNORE_CASE),
+	stringify('UNICODESET_IGNORE_CASE', ESCAPE_CHARS_UNICODESET_IGNORE_CASE)
 ].join('\n\n');
 
 // Save the precompiled sets to a static file.

tests/fixtures/modifiers.js

@@ -161,15 +161,30 @@ const modifiersFixtures = [
 		'pattern': '(?m:^[a-z])',
 		'expected': '(?:(?:^|(?<=[\\n\\r\\u2028\\u2029]))[a-z])',
 	},
+	{
+		'pattern': '(?m:^[a-z])',
+		'options': { modifiers: false },
+		'expected': '(?m:^[a-z])',
+	},
 	{
 		'pattern': '(?m:[a-z]$)',
 		'expected': '(?:[a-z](?:$|(?=[\\n\\r\\u2028\\u2029])))',
 	},
+	{
+		'pattern': '(?m:[a-z]$)',
+		'options': { modifiers: false },
+		'expected': '(?m:[a-z]$)',
+	},
 	// +s
 	{
 		'pattern': '(?s:.)',
 		'expected': '(?:[^])',
 	},
+	{
+		'pattern': '(?s:.)',
+		'options': { modifiers: false },
+		'expected': '(?s:.)',
+	},
 	// -i
 	{
 		'pattern': '(?-i:a)(a)',
@@ -250,6 +265,48 @@ const modifiersFixtures = [
 		'expected': '[A-Za-z\\u017F\\u212A](?:a)',
 		'expectedFlags': 'u'
 	},
+	{
+		'pattern': '(?i:[[AB]&&B])',
+		'options': { unicodeSetsFlag: 'transform', modifiers: 'transform' },
+		'flags': 'v',
+		'expected': '(?:[Bb])'
+	},
+	{
+		'pattern': '(?i:[[AB]&&B])',
+		'options': { modifiers: 'transform' },
+		'flags': 'v',
+		'expected': '(?:[Bb])'
+	},
+	{
+		'pattern': '(?i:[K&&k])',
+		'flags': 'v',
+		'expected': '(?:[Kk\\u212A])',
+		'expectedFlags': 'v'
+	},
+	{
+		'pattern': '(?i:[K--k])',
+		'flags': 'v',
+		'expected': '(?:[])',
+		'expectedFlags': 'v'
+	},
+	{
+		pattern: '(?i:[\\q{KK}&&\\q{kk}])',
+		flags: 'v',
+		expected: '(?:(?:[Kk\\u212A][Kk\\u212A]))',
+		expectedFlags: 'v',
+	},
+	{
+		pattern: '(?i:[\\q{KK}--\\q{k\\u212A}])',
+		flags: 'v',
+		expected: '(?:[])',
+		expectedFlags: 'v'
+	},
+	{
+		pattern: '(?i:[[J-Lj-l]--\\u212A])',
+		flags: 'v',
+		expected: '(?:[JLjl])',
+		expectedFlags: 'v'
+	},
 	// -m
 	{
 		'pattern': '(?-m:^[a-z])(^[a-z])',

tests/fixtures/unicode-set.js

@@ -173,13 +173,15 @@ const unicodeSetFixtures = [
 	{
 		pattern: '[\\q{sA}asb]',
 		flags: 'iv',
-		expected: '(?:sA|[abs])'
+		expected: '(?:sA|[abs])',
+		expectedFlags: 'iu'
 	},
 	{
 		pattern: '[\\q{sA}asb]',
 		flags: 'iv',
 		options: TRANSFORM_U,
-		expected: '(?:[s\\u017F]A|[abs\\u017F])'
+		expected: '(?:[s\\u017F]A|[abs\\u017F])',
+		expectedFlags: 'i'
 	},
 	{
 		pattern: '[[ab\\q{cd}]--a]',
@@ -402,6 +404,148 @@ const unicodeSetFixtures = [
 		flags: 'iv',
 		matches: ['k', 'K', '\u{212A}', '\u{0131}'],
 		nonMatches: ['0', ',']
+	},
+	{
+		pattern: '[K&&k]',
+		flags: 'iv',
+		expected: 'k',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[K&&\\u212A]',
+		flags: 'iv',
+		expected: 'k',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[K--k]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[K--\\q{k}]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\u212A--k]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\q{\\u212A}--k]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[K--\\u212A]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\q{K}--\\q{\\u212A}]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\q{KK}&&\\q{kk}]',
+		flags: 'iv',
+		expected: '(?:kk)',
+		expectedFlags: 'iu',
+	},
+	{
+		pattern: '[\\q{KK}--\\q{k\\u212A}]',
+		flags: 'iv',
+		expected: '[]',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\p{Lu}&&k]',
+		flags: 'iv',
+		expected: 'k',
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\p{Lu}--k]',
+		flags: 'iv',
+		expectedFlags: 'iu',
+		nonMatches: ['K', 'k', '\u212A'],
+	},
+	{
+		pattern: '[[\\p{Lu}]--k]',
+		flags: 'iv',
+		expectedFlags: 'iu',
+		nonMatches: ['K', 'k', '\u212A'],
+	},
+	{
+		pattern: '[\\w--k]',
+		flags: 'iv',
+		expected: '[0-9_a-jl-z]',
+		expectedFlags: 'iu',
+		nonMatches: ['K', 'k', '\u212A'],
+	},
+	{
+		pattern: '[[\\w]--k]',
+		flags: 'iv',
+		expected: '[0-9_a-jl-z]',
+		expectedFlags: 'iu',
+		nonMatches: ['K', 'k', '\u212A'],
+	},
+	{
+		pattern: '[\\W--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[[\\W]--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\D--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[[\\D]--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[\\S--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[[\\S]--Σ]',
+		flags: 'iv',
+		nonMatches: ['Σ', 'σ'],
+		matches: ['Θ', 'θ'],
+		expectedFlags: 'iu'
+	},
+	{
+		pattern: '[[J-Lj-l]--\\u212A]',
+		flags: 'iv',
+		expected: '[jl]',
+		expectedFlags: 'iu',
+		nonMatches: ['K', 'k', '\u212A'],
+		matches: ['j', 'J', 'l', 'L']
 	}
 ];