feat: SSPAPI-332 Optimized uslug

Author: yuriiLevantovych

.idea/codeStyles/Project.xml

@@ -1,21 +1,53 @@
-# xxxxxx [![CircleCI](https://circleci.com/gh/shelfio/xxxxxx/tree/master.svg?style=svg)](https://circleci.com/gh/shelfio/xxxxxx/tree/master)![](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)
+# fast-uslug [![CircleCI](https://circleci.com/gh/shelfio/fast-uslug/tree/master.svg?style=svg)](https://circleci.com/gh/shelfio/fast-uslug/tree/master)![](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)
 
-> xxxxxx description
+> It's a fork of [uslug](https://github.com/jeremys/uslug). **6x** faster than uslug.
+
+Permissive slug generator that works with unicode.
+We keep only characters from the categories Letter, Number and Separator (see [Unicode Categories](http://www.unicode.org/versions/Unicode6.0.0/ch04.pdf))
+and the common [CJK Unified Ideographs](http://www.unicode.org/versions/Unicode6.0.0/ch12.pdf) as defined in the version 6.0.0 of the Unicode specification.
+
+Inspired by [unicode-slugify](https://github.com/mozilla/unicode-slugify).
+Note that this slug generator is different from [node-slug](https://github.com/dodo/node-slug) which focus on translating unicode characters to english or latin equivalent.
+
+## Quick Examples
+
+```js
+fastUslug('Ґатунок Їхніх обценьок неперевершений!'); // 'ґатунок-їхніх-обценьок-неперевершений'
+fastUslug('汉语/漢語'); // '汉语漢語'
+
+fastUslug('Y U NO', {lower: false}); // 'Y-U-NO'
+fastUslug('Y U NO', {spaces: true}); // 'y u no'
+fastUslug('Y-U|NO', {allowedChars: '|'}); // 'yu|no'
+```
 
 ## Install
 
 ```
-$ yarn add @shelf/xxxxxx
+$ yarn add @shelf/fast-uslug
 ```
 
 ## Usage
 
 ```js
-const {getFoo} = require('@shelf/xxxxxx');
+import {fastUslug} from '@shelf/fast-uslug';
 
-getFoo();
+fastUslug('some string'); // some-string
 ```
 
+## Options
+
+### fastUslug(string, options)
+
+Generate a slug for the string passed.
+
+**Arguments**
+
+- string - The string you want to slugify.
+- options - An optional object that can contain:
+  - allowedChars: a Set of chars that you want to be whitelisted. Default: '-\_~'.
+  - lower: a Boolean to force to lower case the slug. Default: true.
+  - spaces: a Boolean to allow spaces. Default: false.
+
 ## Publish
 
 ```sh

.idea/jsLinters/eslint.xml

@@ -1,7 +1,7 @@
 {
-  "name": "@shelf/xxxxxx",
-  "version": "0.0.0",
-  "description": "xxxxxx",
+  "name": "@shelf/fast-uslug",
+  "version": "0.0.1",
+  "description": "fast-uslug",
   "license": "MIT",
   "author": {
     "name": "Vlad Holubiev",
@@ -47,12 +47,14 @@
     "@shelf/tsconfig": "0.0.6",
     "@types/jest": "27.4.0",
     "@types/node": "14",
+    "@types/uslug": "1.0.1",
     "eslint": "8.6.0",
     "husky": "7.0.4",
     "jest": "27.4.7",
     "lint-staged": "12.1.7",
     "prettier": "2.5.1",
-    "typescript": "4.5.4"
+    "typescript": "4.5.4",
+    "uslug": "1.0.4"
   },
   "engines": {
     "node": ">=14"

README.md

@@ -0,0 +1,14 @@
+// List of Unicode code that are flagged as separator.
+//
+// Contains Unicode code of:
+// - Zs = Separator, space
+// - Zl = Separator, line
+// - Zp = Separator, paragraph
+//
+// This list has been computed from http://unicode.org/Public/UNIDATA/UnicodeData.txt
+// curl -s http://unicode.org/Public/UNIDATA/UnicodeData.txt | grep -E ';Zs;|;Zl;|;Zp;' | cut -d \; -f 1 | xargs -I{} printf '%d, ' 0x{}
+//
+export const Z = new Set([
+  32, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8232, 8233, 8239,
+  8287, 12288,
+]);

package.json

@@ -0,0 +1,2 @@
+export * from './LMN';
+export * from './Z';

src/codes/LMN.ts

@@ -0,0 +1,24 @@
+export const normalizeForm = 'NFKC';
+
+export const lmnSymbol = 1;
+export const zSymbol = 2;
+
+export const spaceSymbol = ' ';
+export const emptyString = '';
+export const hyphenSymbol = '-';
+
+export const severalSpacesRegex = /\s+/g;
+export const severalSpacesOrHyphensRegex = /[\s-]+/g;
+
+export const defaultAllowedChars = new Set(['-', '_', '~']);
+
+export const chineseL = 0x4e00;
+export const chineseR = 0x9fff;
+
+export const koreanL = 0xac00;
+export const koreanR = 0xd7a3;
+
+export const japaneseLL = 0x3000;
+export const japaneseLR = 0x3002;
+export const japaneseRL = 0xff01;
+export const japaneseRR = 0xff02;

src/codes/Z.ts

@@ -0,0 +1,73 @@
+import {LMN, Z} from './codes';
+import {
+  chineseL,
+  chineseR,
+  emptyString,
+  japaneseLL,
+  japaneseLR,
+  japaneseRL,
+  japaneseRR,
+  koreanL,
+  koreanR,
+  lmnSymbol,
+  spaceSymbol,
+  zSymbol,
+} from './consts';
+
+export const getRawSlug = (chars: string, allowedChars: Set<string>): string => {
+  const rawSlug = [];
+
+  for (const char of chars) {
+    const code = char.charCodeAt(0);
+
+    if (allowChinese(code) || allowKorean(code)) {
+      rawSlug.push(char);
+      continue;
+    }
+
+    if (allowJapanese(code)) {
+      rawSlug.push(spaceSymbol);
+    }
+
+    if (allowedChars.has(char)) {
+      rawSlug.push(char);
+      continue;
+    }
+
+    const value = unicodeCategory(code);
+
+    if (!value) {
+      continue;
+    }
+
+    rawSlug.push(value === lmnSymbol ? char : spaceSymbol);
+  }
+
+  return rawSlug.join(emptyString).trim();
+};
+
+// Allow Common CJK Unified Ideographs
+// See: http://www.unicode.org/versions/Unicode6.0.0/ch12.pdf - Table 12-2
+function allowChinese(code: number): boolean {
+  return chineseL <= code && code <= chineseR;
+}
+
+// Allow Hangul
+function allowKorean(code: number): boolean {
+  return koreanL <= code && code <= koreanR;
+}
+
+// Japanese ideographic punctuation
+function allowJapanese(code: number): boolean {
+  return (japaneseLL <= code && code <= japaneseLR) || (japaneseRL <= code && code <= japaneseRR);
+}
+
+function unicodeCategory(code: number): undefined | number {
+  if (LMN.has(code)) {
+    return lmnSymbol;
+  }
+
+  if (Z.has(code)) {
+    return zSymbol;
+  }
+}

src/codes/index.ts

@@ -0,0 +1,45 @@
+import uslug from 'uslug';
+import mocks from './mocks';
+import {fastUslug} from './index';
+
+const runTimes = 10;
+const desirableRunTimes = 1000;
+const testsIncreaseTimes = Math.floor(desirableRunTimes / mocks.length);
+const desirableTests = Array(testsIncreaseTimes).fill(mocks).flat();
+
+describe.skip('old-new-uslug-benchmark', () => {
+  let sumFaster = 0;
+
+  afterAll(() => {
+    console.log(`In average fast-uslug ${(sumFaster / runTimes).toFixed(2)}x faster than uslug`);
+  });
+
+  for (let i = 0; i < runTimes; i++) {
+    describe('old-new-uslug', () => {
+      let spentTimeOld: number;
+      let spentTimeNew: number;
+
+      afterAll(() => {
+        sumFaster += spentTimeOld / spentTimeNew;
+      });
+
+      it.concurrent('should old slugify', async () => {
+        const startOld = +new Date();
+        for (const test of desirableTests) {
+          expect(uslug(test[0])).toEqual(test[1]);
+        }
+        const endOld = +new Date();
+        spentTimeOld = endOld - startOld;
+      });
+
+      it.concurrent('should new slugify', async () => {
+        const startNew = +new Date();
+        for (const test of desirableTests) {
+          expect(fastUslug(test[0])).toEqual(test[1]);
+        }
+        const endNew = +new Date();
+        spentTimeNew = endNew - startNew;
+      });
+    });
+  }
+});

src/consts.ts

@@ -1,5 +1,26 @@
-import {getFoo} from './index';
+import mocks from './mocks';
+import {fastUslug} from './index';
 
-it('should return bar', () => {
-  expect(getFoo()).toEqual('bar');
+it('should return slugified strings', () => {
+  for (const mock of mocks) {
+    expect(fastUslug(mock[0])).toEqual(mock[1]);
+  }
+});
+
+it('should return default allowed chars', () => {
+  expect(fastUslug('~-_')).toEqual('~-_');
+});
+
+it('should leave passed allowed chars', () => {
+  expect(fastUslug('qwerty |@ qwerty', {allowedChars: new Set(['|', '@'])})).toEqual(
+    'qwerty-|@-qwerty'
+  );
+});
+
+it('should not be lower cased', () => {
+  expect(fastUslug('QWERTY', {lower: false})).toEqual('QWERTY');
+});
+
+it('should leave one space between words', () => {
+  expect(fastUslug(' qwerty   qwerty ', {spaces: true})).toEqual('qwerty qwerty');
 });

src/helpers.ts

@@ -1,3 +1,32 @@
-export function getFoo(): string {
-  return 'bar';
-}
+import {
+  defaultAllowedChars,
+  emptyString,
+  hyphenSymbol,
+  normalizeForm,
+  severalSpacesOrHyphensRegex,
+  severalSpacesRegex,
+  spaceSymbol,
+} from './consts';
+import {getRawSlug} from './helpers';
+
+type OptionParams = {lower?: boolean; spaces?: boolean; allowedChars?: Set<string>};
+
+export const fastUslug = (
+  str: string = emptyString,
+  {lower = true, spaces = false, allowedChars = defaultAllowedChars}: OptionParams = {}
+): string => {
+  const chars = str.normalize(normalizeForm);
+  const rawSlug = getRawSlug(chars, allowedChars);
+
+  let slug = rawSlug.replace(severalSpacesRegex, spaceSymbol);
+
+  if (!spaces) {
+    slug = slug.replace(severalSpacesOrHyphensRegex, hyphenSymbol);
+  }
+
+  if (lower) {
+    slug = slug.toLowerCase();
+  }
+
+  return slug;
+};

src/index.benchmark.test.ts

@@ -0,0 +1,34 @@
+export default [
+  [
+    'The \u212B symbol invented by A. J. \u00C5ngstr\u00F6m (1814, L\u00F6gd\u00F6, \u2013 1874) denotes the length 10\u207B\u00B9\u2070 m.',
+    'the-å-symbol-invented-by-a-j-ångström-1814-lögdö-1874-denotes-the-length-1010-m',
+  ],
+  ['Ґатунок Їхніх обценьок неперевершений!', 'ґатунок-їхніх-обценьок-неперевершений'],
+  ['xx x  - "#$@ x', 'xx-x-x'],
+  ['Bän...g (bang)', 'bäng-bang'],
+  ['    a ', 'a'],
+  ['tags/', 'tags'],
+  ['y_u_no', 'y_u_no'],
+  ['el-ni\xf1o', 'el-ni\xf1o'],
+  ['¿x', 'x'],
+  ['فار,سي', 'فارسي'],
+  ['เแโ|ใไ', 'เแโใไ'],
+  ['संसद में काम नहीं तो वेतन क्यों?', 'संसद-में-काम-नहीं-तो-वेतन-क्यों'],
+  [
+    "เร่งรัด 'ปรับเงินเดือนท้องถิ่น 1 ขั้น' ตามมติ ครม.",
+    'เร่งรัด-ปรับเงินเดือนท้องถิ่น-1-ขั้น-ตามมติ-ครม',
+  ],
+  ['Ελληνικά - Ελληνικά', 'ελληνικά-ελληνικά'],
+  ['汉语/漢語', '汉语漢語'],
+  ['ϧ΃蒬蓣', '\u03e7蒬蓣'],
+  ['一二三四五六七八九十！。。。', '一二三四五六七八九十'],
+  ['日本語ドキュメンテ(ーション)', '日本語ドキュメンテーション'],
+  [
+    'オバマ大統領が病院爆撃の調査へ同意するように、協力してください！',
+    'オバマ大統領が病院爆撃の調査へ同意するように-協力してください',
+  ],
+  [
+    '일본정부 법무대신(法務大臣): 우리는 일본 입관법의 재검토를 요구한다!',
+    '일본정부-법무대신法務大臣-우리는-일본-입관법의-재검토를-요구한다',
+  ],
+];