[READ-ONLY] Mirror of https://github.com/FoxxMD/string-sameness. Compare the sameness of two strings
foxxmd.github.io/string-sameness/
compare
cosine-similarity
dice-coefficient
levenshtein-distance
sameness
string
text
typescript
fix: Fix punctuation transform removing non-latin letter/characters
* Use a stricter regex to remove only common punctuation/symbols which fixes accidental removal of non-latin letter-characters
* Provide non-alphanumeric removal as an exported transform
+4
-1
+4
-1
src/normalization/index.ts
+4
-1
src/normalization/index.ts
···
1
1
import {StringTransformFunc} from "../atomic.js";
2
2
3
-
const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g);
3
+
const PUNCTUATION_REGEX = new RegExp(/[`=(){}<>;',.~!@#$%^&*_+|:"?\-\\\[\]\/]/g);
4
+
const NON_ALPHANUMERIC_REGEX = new RegExp(/[^\w\s]|_/g);
4
5
const WHITESPACE_REGEX = new RegExp(/\s/g);
5
6
const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g);
6
7
···
8
9
const trim: StringTransformFunc = (str: string) => str.trim();
9
10
const replaceUnicode: StringTransformFunc = (str: string) => str.normalize('NFD').replace(/[\u0300-\u036f]/g, "");
10
11
const removePunctuation: StringTransformFunc = (str: string) => str.replace(PUNCTUATION_REGEX, '');
12
+
const removeNonAlphanumeric: StringTransformFunc = (str: string) => str.replace(NON_ALPHANUMERIC_REGEX, '');
11
13
const removeWhitespace: StringTransformFunc = (str: string) => str.replace(WHITESPACE_REGEX, '');
12
14
const replaceMultiWhitespace: StringTransformFunc = (str: string) => str.replace(MULTI_WHITESPACE_REGEX, ' ');
13
15
···
34
36
replaceUnicode,
35
37
removePunctuation,
36
38
removeWhitespace,
39
+
removeNonAlphanumeric,
37
40
replaceMultiWhitespace,
38
41
transforms,
39
42
strDefaultTransforms