foxxmd.dev / string-sameness
[READ-ONLY] Mirror of https://github.com/FoxxMD/string-sameness. Compare the sameness of two strings foxxmd.github.io/string-sameness/
compare cosine-similarity dice-coefficient levenshtein-distance sameness string text typescript
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

Update dist

FoxxMD (Jan 30, 2024, 3:57 PM EST) 950b8b78 5b0568c1

+135 -56
+8
dist/commonjs/atomic.d.ts
··· 13 13 * Useful when only the differences in content are important, but not the order of the content 14 14 * */ 15 15 reorder?: boolean; 16 + /** 17 + * When `reorder` is used this determines how to split each string into the tokens that will be reordered. 18 + * 19 + * The value of this property is used in String.split() -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/split#separator 20 + * 21 + * @default " " 22 + * */ 23 + delimiter?: string | RegExp; 16 24 } 17 25 export interface StringSamenessResult { 18 26 strategies: {
+1 -1
dist/commonjs/index.d.ts
··· 2 2 import { strDefaultTransforms, transforms } from "./normalization/index.js"; 3 3 declare const defaultStrategies: import("./atomic.js").ComparisonStrategy<import("./atomic.js").ComparisonStrategyResultObject>[]; 4 4 declare const stringSameness: (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; 5 - export declare const reorderStr: (cleanA: string, cleanB: string, options?: StringComparisonOptions) => string; 5 + export declare const reorderStr: (strA: string, strB: string, options?: StringComparisonOptions) => [string, string]; 6 6 declare const createStringSameness: (defaults: StringComparisonOptions) => (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; 7 7 declare const strategies: { 8 8 diceStrategy: import("./atomic.js").ComparisonStrategy<import("./atomic.js").ComparisonStrategyResultObject>;
+50 -22
dist/commonjs/index.js
··· 17 17 ]; 18 18 exports.defaultStrategies = defaultStrategies; 19 19 const stringSameness = (valA, valB, options) => { 20 - const { transforms = index_js_2.strDefaultTransforms, strategies = defaultStrategies, reorder = false, } = options || {}; 21 - const cleanA = transforms.reduce((acc, curr) => curr(acc), valA); 20 + const { transforms = index_js_2.strDefaultTransforms, strategies = defaultStrategies, reorder = false, delimiter = ' ' } = options || {}; 21 + let cleanA = transforms.reduce((acc, curr) => curr(acc), valA); 22 22 let cleanB = transforms.reduce((acc, curr) => curr(acc), valB); 23 - const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; 24 23 if (reorder) { 25 24 // we want to ignore order of tokens as much as possible (user does not care about differences in word order, just absolute differences in characters overall) 26 - // so we will reorder cleanB so its tokens match the order or tokens in cleanA as closely as possible 25 + // so we will reorder the shorter of the two strings so its tokens match the order of tokens in the longer string as closely as possible 27 26 // before we run strategies 28 - cleanB = (0, exports.reorderStr)(cleanA, cleanB); 27 + const [orderedX, orderedY] = (0, exports.reorderStr)(cleanA, cleanB); 28 + cleanA = orderedX; 29 + cleanB = orderedY; 29 30 } 31 + const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; 30 32 const stratResults = []; 31 33 for (const strat of strategies) { 32 34 if (strat.isValid !== undefined && !strat.isValid(cleanA, cleanB)) { ··· 60 62 }; 61 63 }; 62 64 exports.stringSameness = stringSameness; 63 - const reorderStr = (cleanA, cleanB, options) => { 64 - // to do the reordering we will use stringSameness with the provided strats to match against each token in cleanA and choose the closest token in cleanB 65 - // and add the end concat any remaining tokens from cleanB to the reordered string 66 - const aTokens = cleanA.split(' '); 67 - const bTokens = cleanB.split(' '); 68 - const orderedCandidateTokens = aTokens.reduce((acc, curr) => { 65 + const reorderStr = (strA, strB, options) => { 66 + const { transforms = index_js_2.strDefaultTransforms, strategies = defaultStrategies, delimiter = ' ' } = options || {}; 67 + const cleanA = transforms.reduce((acc, curr) => curr(acc), strA); 68 + const cleanB = transforms.reduce((acc, curr) => curr(acc), strB); 69 + // split by "token" 70 + const eTokens = cleanA.split(delimiter); 71 + const cTokens = cleanB.split(delimiter); 72 + let longerTokens, shorterTokens; 73 + if (eTokens.length > cTokens.length) { 74 + longerTokens = eTokens; 75 + shorterTokens = cTokens; 76 + } 77 + else { 78 + longerTokens = cTokens; 79 + shorterTokens = eTokens; 80 + } 81 + // we will use longest string (token list) as the reducer and order the shorter list to match it 82 + // so we don't have to deal with undefined positions in the shorter list 83 + const orderedCandidateTokens = longerTokens.reduce((acc, curr) => { 84 + // if we've run out of tokens in the shorter list just return 85 + if (acc.remaining.length === 0) { 86 + return acc; 87 + } 88 + // on each iteration of tokens in the long list 89 + // we iterate through remaining tokens from the shorter list and find the token with the most sameness 69 90 let highScore = 0; 70 91 let highIndex = 0; 71 92 let index = 0; 72 93 for (const token of acc.remaining) { 73 - const result = stringSameness(curr, token, { ...options, reorder: false }); 74 - if (result.highScore > highScore) { 75 - highScore = result.highScore; 94 + const result = stringSameness(curr, token, { strategies }); 95 + if (result.highScoreWeighted > highScore) { 96 + highScore = result.highScoreWeighted; 76 97 highIndex = index; 77 98 } 78 99 index++; 79 100 } 101 + // then remove the most same token from the remaining short list tokens 80 102 const splicedRemaining = [...acc.remaining]; 81 - if (highIndex <= splicedRemaining.length - 1) { 82 - splicedRemaining.splice(highIndex, 1); 83 - } 84 - const ordered = highIndex <= acc.remaining.length - 1 ? acc.ordered.concat(acc.remaining[highIndex]) : acc.ordered; 85 - return { ordered: ordered, remaining: splicedRemaining }; 86 - }, { ordered: [], remaining: bTokens }); 87 - const allOrderedCandidateTokens = orderedCandidateTokens.ordered.concat(orderedCandidateTokens.remaining); 88 - return allOrderedCandidateTokens.join(' '); 103 + splicedRemaining.splice(highIndex, 1); 104 + return { 105 + // finally add the most same token to the ordered short list 106 + ordered: acc.ordered.concat(acc.remaining[highIndex]), 107 + // and return the remaining short list tokens 108 + remaining: splicedRemaining 109 + }; 110 + }, { 111 + // "ordered" is the result of ordering tokens in the shorter list to match longer token order 112 + ordered: [], 113 + // remaining is the initial shorter list 114 + remaining: shorterTokens 115 + }); 116 + return [longerTokens.join(' '), orderedCandidateTokens.ordered.join(' ')]; 89 117 }; 90 118 exports.reorderStr = reorderStr; 91 119 const createStringSameness = (defaults) => {
+1 -1
dist/commonjs/index.js.map
··· 1 - {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,4DAAoH;AAQpH,uDAA0E;AA2HtE,qGA3HI,+BAAoB,OA2HJ;AAFpB,2FAzH0B,qBAAU,OAyH1B;AAvHd,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,uBAAY;IACZ,wBAAa;IACb,yBAAc;CACjB,CAAA;AA2GG,8CAAiB;AAzGrB,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,+BAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,OAAO,GAAG,KAAK,GAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,IAAI,OAAO,EAAE;QACT,8JAA8J;QAC9J,qGAAqG;QACrG,2BAA2B;QAC3B,MAAM,GAAG,IAAA,kBAAU,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACvC;IAED,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AAiDG,wCAAc;AA/CX,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,MAAc,EAAE,OAAiC,EAAU,EAAE;IACpG,wJAAwJ;IACxJ,kFAAkF;IAClF,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAA+C,EAAE,IAAI,EAAE,EAAE;QACpG,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAAS,GAAuB,CAAC,CAAC;QACtC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,SAAS,EAAE;YAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAC,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;YACzE,IAAI,MAAM,CAAC,SAAS,GAAG,SAAS,EAAE;gBAC9B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC7B,SAAS,GAAG,KAAK,CAAC;aACrB;YACD,KAAK,EAAE,CAAC;SACX;QAED,MAAM,gBAAgB,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAG,SAAS,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;YACzC,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,MAAM,OAAO,GAAG,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;QAEnH,OAAO,EAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAC,CAAC;IAC3D,CAAC,EAAE,EAAC,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAC,CAAC,CAAC;IACtC,MAAM,yBAAyB,GAAG,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAC1G,OAAO,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC,CAAA;AA5BY,QAAA,UAAU,cA4BtB;AAED,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAgBG,oDAAoB;AAdxB,MAAM,UAAU,GAAG;IACf,YAAY,EAAZ,uBAAY;IACZ,aAAa,EAAb,wBAAa;IACb,cAAc,EAAd,yBAAc;IACd,wBAAwB,EAAxB,mCAAwB;CAC3B,CAAC;AAWE,gCAAU;AATd,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,+BAAoB,CAAC;AAUzD,0EAA+B"} 1 + {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,4DAAoH;AAQpH,uDAA0E;AAuKtE,qGAvKI,+BAAoB,OAuKJ;AAFpB,2FArK0B,qBAAU,OAqK1B;AAnKd,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,uBAAY;IACZ,wBAAa;IACb,yBAAc;CACjB,CAAA;AAuJG,8CAAiB;AArJrB,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,+BAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,OAAO,GAAG,KAAK,EACf,SAAS,GAAG,GAAG,EAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAE/D,IAAI,OAAO,EAAE;QACT,8JAA8J;QAC9J,wIAAwI;QACxI,2BAA2B;QAC3B,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAA,kBAAU,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,GAAG,QAAQ,CAAC;QAClB,MAAM,GAAG,QAAQ,CAAC;KACrB;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AA0FG,wCAAc;AAxFX,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAoB,EAAE;IAE1G,MAAM,EACF,UAAU,GAAG,+BAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,SAAS,GAAG,GAAG,EAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAEjE,mBAAmB;IACnB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAGxC,IAAI,YAAsB,EACtB,aAAuB,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE;QACjC,YAAY,GAAG,OAAO,CAAC;QACvB,aAAa,GAAG,OAAO,CAAC;KAC3B;SAAM;QACH,YAAY,GAAG,OAAO,CAAC;QACvB,aAAa,GAAG,OAAO,CAAC;KAC3B;IAED,gGAAgG;IAChG,wEAAwE;IAExE,MAAM,sBAAsB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAA+C,EAAE,IAAI,EAAE,EAAE;QACzG,6DAA6D;QAC7D,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC5B,OAAO,GAAG,CAAC;SACd;QAED,+CAA+C;QAC/C,sGAAsG;QAEtG,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,SAAS,EAAE;YAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC;YACzD,IAAI,MAAM,CAAC,iBAAiB,GAAG,SAAS,EAAE;gBACtC,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBACrC,SAAS,GAAG,KAAK,CAAC;aACrB;YACD,KAAK,EAAE,CAAC;SACX;QAED,uEAAuE;QACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAEtC,OAAO;YACH,4DAA4D;YAC5D,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACrD,6CAA6C;YAC7C,SAAS,EAAE,gBAAgB;SAC9B,CAAC;IACN,CAAC,EAAE;QACC,6FAA6F;QAC7F,OAAO,EAAE,EAAE;QACX,wCAAwC;QACxC,SAAS,EAAE,aAAa;KAC3B,CAAC,CAAC;IAEH,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9E,CAAC,CAAA;AArEY,QAAA,UAAU,cAqEtB;AAED,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAgBG,oDAAoB;AAdxB,MAAM,UAAU,GAAG;IACf,YAAY,EAAZ,uBAAY;IACZ,aAAa,EAAb,wBAAa;IACb,cAAc,EAAd,yBAAc;IACd,wBAAwB,EAAxB,mCAAwB;CAC3B,CAAC;AAWE,gCAAU;AATd,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,+BAAoB,CAAC;AAUzD,0EAA+B"}
+2 -1
dist/commonjs/normalization/index.d.ts
··· 3 3 declare const trim: StringTransformFunc; 4 4 declare const replaceUnicode: StringTransformFunc; 5 5 declare const removePunctuation: StringTransformFunc; 6 + declare const removeNonAlphanumeric: StringTransformFunc; 6 7 declare const removeWhitespace: StringTransformFunc; 7 8 declare const replaceMultiWhitespace: StringTransformFunc; 8 9 declare const transforms: { ··· 14 15 removePunctuation: StringTransformFunc; 15 16 }; 16 17 declare const strDefaultTransforms: StringTransformFunc[]; 17 - export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; 18 + export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, removeNonAlphanumeric, replaceMultiWhitespace, transforms, strDefaultTransforms };
+5 -2
dist/commonjs/normalization/index.js
··· 1 1 "use strict"; 2 2 Object.defineProperty(exports, "__esModule", { value: true }); 3 - exports.strDefaultTransforms = exports.transforms = exports.replaceMultiWhitespace = exports.removeWhitespace = exports.removePunctuation = exports.replaceUnicode = exports.trim = exports.lowercase = void 0; 4 - const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g); 3 + exports.strDefaultTransforms = exports.transforms = exports.replaceMultiWhitespace = exports.removeNonAlphanumeric = exports.removeWhitespace = exports.removePunctuation = exports.replaceUnicode = exports.trim = exports.lowercase = void 0; 4 + const PUNCTUATION_REGEX = new RegExp(/[`=(){}<>;',.~!@#$%^&*_+|:"?\-\\\[\]\/]/g); 5 + const NON_ALPHANUMERIC_REGEX = new RegExp(/[^\w\s]|_/g); 5 6 const WHITESPACE_REGEX = new RegExp(/\s/g); 6 7 const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g); 7 8 const lowercase = (str) => str.toLocaleLowerCase(); ··· 12 13 exports.replaceUnicode = replaceUnicode; 13 14 const removePunctuation = (str) => str.replace(PUNCTUATION_REGEX, ''); 14 15 exports.removePunctuation = removePunctuation; 16 + const removeNonAlphanumeric = (str) => str.replace(NON_ALPHANUMERIC_REGEX, ''); 17 + exports.removeNonAlphanumeric = removeNonAlphanumeric; 15 18 const removeWhitespace = (str) => str.replace(WHITESPACE_REGEX, ''); 16 19 exports.removeWhitespace = removeWhitespace; 17 20 const replaceMultiWhitespace = (str) => str.replace(MULTI_WHITESPACE_REGEX, ' ');
+1 -1
dist/commonjs/normalization/index.js.map
··· 1 - {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":";;;AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACnD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;AAErD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AAyB5E,8BAAS;AAxBb,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAyB1D,oBAAI;AAxBR,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AAyB9G,wCAAc;AAxBlB,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAyB/F,8CAAiB;AAxBrB,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AAyB7F,4CAAgB;AAxBpB,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;AAyB1G,wDAAsB;AAvB1B,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAiBG,gCAAU;AAfd,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAUE,oDAAoB"} 1 + {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":";;;AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,0CAA0C,CAAC,CAAC;AACjF,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACxD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;AAErD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AA0B5E,8BAAS;AAzBb,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AA0B1D,oBAAI;AAzBR,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AA0B9G,wCAAc;AAzBlB,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AA0B/F,8CAAiB;AAzBrB,MAAM,qBAAqB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;AA2BxG,sDAAqB;AA1BzB,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AAyB7F,4CAAgB;AAxBpB,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;AA0B1G,wDAAsB;AAxB1B,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAkBG,gCAAU;AAhBd,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAWE,oDAAoB"}
+8
dist/esm/atomic.d.ts
··· 13 13 * Useful when only the differences in content are important, but not the order of the content 14 14 * */ 15 15 reorder?: boolean; 16 + /** 17 + * When `reorder` is used this determines how to split each string into the tokens that will be reordered. 18 + * 19 + * The value of this property is used in String.split() -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/split#separator 20 + * 21 + * @default " " 22 + * */ 23 + delimiter?: string | RegExp; 16 24 } 17 25 export interface StringSamenessResult { 18 26 strategies: {
+1 -1
dist/esm/index.d.ts
··· 2 2 import { strDefaultTransforms, transforms } from "./normalization/index.js"; 3 3 declare const defaultStrategies: import("./atomic.js").ComparisonStrategy<import("./atomic.js").ComparisonStrategyResultObject>[]; 4 4 declare const stringSameness: (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; 5 - export declare const reorderStr: (cleanA: string, cleanB: string, options?: StringComparisonOptions) => string; 5 + export declare const reorderStr: (strA: string, strB: string, options?: StringComparisonOptions) => [string, string]; 6 6 declare const createStringSameness: (defaults: StringComparisonOptions) => (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; 7 7 declare const strategies: { 8 8 diceStrategy: import("./atomic.js").ComparisonStrategy<import("./atomic.js").ComparisonStrategyResultObject>;
+50 -22
dist/esm/index.js
··· 11 11 cosineStrategy 12 12 ]; 13 13 const stringSameness = (valA, valB, options) => { 14 - const { transforms = strDefaultTransforms, strategies = defaultStrategies, reorder = false, } = options || {}; 15 - const cleanA = transforms.reduce((acc, curr) => curr(acc), valA); 14 + const { transforms = strDefaultTransforms, strategies = defaultStrategies, reorder = false, delimiter = ' ' } = options || {}; 15 + let cleanA = transforms.reduce((acc, curr) => curr(acc), valA); 16 16 let cleanB = transforms.reduce((acc, curr) => curr(acc), valB); 17 - const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; 18 17 if (reorder) { 19 18 // we want to ignore order of tokens as much as possible (user does not care about differences in word order, just absolute differences in characters overall) 20 - // so we will reorder cleanB so its tokens match the order or tokens in cleanA as closely as possible 19 + // so we will reorder the shorter of the two strings so its tokens match the order of tokens in the longer string as closely as possible 21 20 // before we run strategies 22 - cleanB = reorderStr(cleanA, cleanB); 21 + const [orderedX, orderedY] = reorderStr(cleanA, cleanB); 22 + cleanA = orderedX; 23 + cleanB = orderedY; 23 24 } 25 + const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; 24 26 const stratResults = []; 25 27 for (const strat of strategies) { 26 28 if (strat.isValid !== undefined && !strat.isValid(cleanA, cleanB)) { ··· 53 55 highScoreWeighted, 54 56 }; 55 57 }; 56 - export const reorderStr = (cleanA, cleanB, options) => { 57 - // to do the reordering we will use stringSameness with the provided strats to match against each token in cleanA and choose the closest token in cleanB 58 - // and add the end concat any remaining tokens from cleanB to the reordered string 59 - const aTokens = cleanA.split(' '); 60 - const bTokens = cleanB.split(' '); 61 - const orderedCandidateTokens = aTokens.reduce((acc, curr) => { 58 + export const reorderStr = (strA, strB, options) => { 59 + const { transforms = strDefaultTransforms, strategies = defaultStrategies, delimiter = ' ' } = options || {}; 60 + const cleanA = transforms.reduce((acc, curr) => curr(acc), strA); 61 + const cleanB = transforms.reduce((acc, curr) => curr(acc), strB); 62 + // split by "token" 63 + const eTokens = cleanA.split(delimiter); 64 + const cTokens = cleanB.split(delimiter); 65 + let longerTokens, shorterTokens; 66 + if (eTokens.length > cTokens.length) { 67 + longerTokens = eTokens; 68 + shorterTokens = cTokens; 69 + } 70 + else { 71 + longerTokens = cTokens; 72 + shorterTokens = eTokens; 73 + } 74 + // we will use longest string (token list) as the reducer and order the shorter list to match it 75 + // so we don't have to deal with undefined positions in the shorter list 76 + const orderedCandidateTokens = longerTokens.reduce((acc, curr) => { 77 + // if we've run out of tokens in the shorter list just return 78 + if (acc.remaining.length === 0) { 79 + return acc; 80 + } 81 + // on each iteration of tokens in the long list 82 + // we iterate through remaining tokens from the shorter list and find the token with the most sameness 62 83 let highScore = 0; 63 84 let highIndex = 0; 64 85 let index = 0; 65 86 for (const token of acc.remaining) { 66 - const result = stringSameness(curr, token, { ...options, reorder: false }); 67 - if (result.highScore > highScore) { 68 - highScore = result.highScore; 87 + const result = stringSameness(curr, token, { strategies }); 88 + if (result.highScoreWeighted > highScore) { 89 + highScore = result.highScoreWeighted; 69 90 highIndex = index; 70 91 } 71 92 index++; 72 93 } 94 + // then remove the most same token from the remaining short list tokens 73 95 const splicedRemaining = [...acc.remaining]; 74 - if (highIndex <= splicedRemaining.length - 1) { 75 - splicedRemaining.splice(highIndex, 1); 76 - } 77 - const ordered = highIndex <= acc.remaining.length - 1 ? acc.ordered.concat(acc.remaining[highIndex]) : acc.ordered; 78 - return { ordered: ordered, remaining: splicedRemaining }; 79 - }, { ordered: [], remaining: bTokens }); 80 - const allOrderedCandidateTokens = orderedCandidateTokens.ordered.concat(orderedCandidateTokens.remaining); 81 - return allOrderedCandidateTokens.join(' '); 96 + splicedRemaining.splice(highIndex, 1); 97 + return { 98 + // finally add the most same token to the ordered short list 99 + ordered: acc.ordered.concat(acc.remaining[highIndex]), 100 + // and return the remaining short list tokens 101 + remaining: splicedRemaining 102 + }; 103 + }, { 104 + // "ordered" is the result of ordering tokens in the shorter list to match longer token order 105 + ordered: [], 106 + // remaining is the initial shorter list 107 + remaining: shorterTokens 108 + }); 109 + return [longerTokens.join(' '), orderedCandidateTokens.ordered.join(' ')]; 82 110 }; 83 111 const createStringSameness = (defaults) => { 84 112 return (valA, valB, options = {}) => stringSameness(valA, valB, { ...defaults, ...options });
+1 -1
dist/esm/index.js.map
··· 1 - {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,wBAAwB,EAAC,MAAM,+BAA+B,CAAC;AAQpH,OAAO,EAAC,oBAAoB,EAAE,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAE1E,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,YAAY;IACZ,aAAa;IACb,cAAc;CACjB,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,oBAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,OAAO,GAAG,KAAK,GAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,IAAI,OAAO,EAAE;QACT,8JAA8J;QAC9J,qGAAqG;QACrG,2BAA2B;QAC3B,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACvC;IAED,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,MAAc,EAAE,OAAiC,EAAU,EAAE;IACpG,wJAAwJ;IACxJ,kFAAkF;IAClF,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAA+C,EAAE,IAAI,EAAE,EAAE;QACpG,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAAS,GAAuB,CAAC,CAAC;QACtC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,SAAS,EAAE;YAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAC,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;YACzE,IAAI,MAAM,CAAC,SAAS,GAAG,SAAS,EAAE;gBAC9B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC7B,SAAS,GAAG,KAAK,CAAC;aACrB;YACD,KAAK,EAAE,CAAC;SACX;QAED,MAAM,gBAAgB,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAG,SAAS,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;YACzC,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,MAAM,OAAO,GAAG,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;QAEnH,OAAO,EAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAC,CAAC;IAC3D,CAAC,EAAE,EAAC,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAC,CAAC,CAAC;IACtC,MAAM,yBAAyB,GAAG,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAC1G,OAAO,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAED,MAAM,UAAU,GAAG;IACf,YAAY;IACZ,aAAa;IACb,cAAc;IACd,wBAAwB;CAC3B,CAAC;AAEF,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,oBAAoB,CAAC;AAE7D,OAAO,EAGH,cAAc,EACd,oBAAoB,EACpB,iBAAiB,EACjB,UAAU,EACV,UAAU,EACV,+BAA+B,EAC/B,oBAAoB,EAGvB,CAAA"} 1 + {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,wBAAwB,EAAC,MAAM,+BAA+B,CAAC;AAQpH,OAAO,EAAC,oBAAoB,EAAE,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAE1E,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,YAAY;IACZ,aAAa;IACb,cAAc;CACjB,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,oBAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,OAAO,GAAG,KAAK,EACf,SAAS,GAAG,GAAG,EAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAE/D,IAAI,OAAO,EAAE;QACT,8JAA8J;QAC9J,wIAAwI;QACxI,2BAA2B;QAC3B,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,GAAG,QAAQ,CAAC;QAClB,MAAM,GAAG,QAAQ,CAAC;KACrB;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAoB,EAAE;IAE1G,MAAM,EACF,UAAU,GAAG,oBAAoB,EACjC,UAAU,GAAG,iBAAiB,EAC9B,SAAS,GAAG,GAAG,EAClB,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAEjE,mBAAmB;IACnB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAGxC,IAAI,YAAsB,EACtB,aAAuB,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE;QACjC,YAAY,GAAG,OAAO,CAAC;QACvB,aAAa,GAAG,OAAO,CAAC;KAC3B;SAAM;QACH,YAAY,GAAG,OAAO,CAAC;QACvB,aAAa,GAAG,OAAO,CAAC;KAC3B;IAED,gGAAgG;IAChG,wEAAwE;IAExE,MAAM,sBAAsB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAA+C,EAAE,IAAI,EAAE,EAAE;QACzG,6DAA6D;QAC7D,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC5B,OAAO,GAAG,CAAC;SACd;QAED,+CAA+C;QAC/C,sGAAsG;QAEtG,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,SAAS,EAAE;YAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC;YACzD,IAAI,MAAM,CAAC,iBAAiB,GAAG,SAAS,EAAE;gBACtC,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBACrC,SAAS,GAAG,KAAK,CAAC;aACrB;YACD,KAAK,EAAE,CAAC;SACX;QAED,uEAAuE;QACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5C,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAEtC,OAAO;YACH,4DAA4D;YAC5D,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACrD,6CAA6C;YAC7C,SAAS,EAAE,gBAAgB;SAC9B,CAAC;IACN,CAAC,EAAE;QACC,6FAA6F;QAC7F,OAAO,EAAE,EAAE;QACX,wCAAwC;QACxC,SAAS,EAAE,aAAa;KAC3B,CAAC,CAAC;IAEH,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9E,CAAC,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAED,MAAM,UAAU,GAAG;IACf,YAAY;IACZ,aAAa;IACb,cAAc;IACd,wBAAwB;CAC3B,CAAC;AAEF,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,oBAAoB,CAAC;AAE7D,OAAO,EAGH,cAAc,EACd,oBAAoB,EACpB,iBAAiB,EACjB,UAAU,EACV,UAAU,EACV,+BAA+B,EAC/B,oBAAoB,EAGvB,CAAA"}
+2 -1
dist/esm/normalization/index.d.ts
··· 3 3 declare const trim: StringTransformFunc; 4 4 declare const replaceUnicode: StringTransformFunc; 5 5 declare const removePunctuation: StringTransformFunc; 6 + declare const removeNonAlphanumeric: StringTransformFunc; 6 7 declare const removeWhitespace: StringTransformFunc; 7 8 declare const replaceMultiWhitespace: StringTransformFunc; 8 9 declare const transforms: { ··· 14 15 removePunctuation: StringTransformFunc; 15 16 }; 16 17 declare const strDefaultTransforms: StringTransformFunc[]; 17 - export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; 18 + export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, removeNonAlphanumeric, replaceMultiWhitespace, transforms, strDefaultTransforms };
+4 -2
dist/esm/normalization/index.js
··· 1 - const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g); 1 + const PUNCTUATION_REGEX = new RegExp(/[`=(){}<>;',.~!@#$%^&*_+|:"?\-\\\[\]\/]/g); 2 + const NON_ALPHANUMERIC_REGEX = new RegExp(/[^\w\s]|_/g); 2 3 const WHITESPACE_REGEX = new RegExp(/\s/g); 3 4 const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g); 4 5 const lowercase = (str) => str.toLocaleLowerCase(); 5 6 const trim = (str) => str.trim(); 6 7 const replaceUnicode = (str) => str.normalize('NFD').replace(/[\u0300-\u036f]/g, ""); 7 8 const removePunctuation = (str) => str.replace(PUNCTUATION_REGEX, ''); 9 + const removeNonAlphanumeric = (str) => str.replace(NON_ALPHANUMERIC_REGEX, ''); 8 10 const removeWhitespace = (str) => str.replace(WHITESPACE_REGEX, ''); 9 11 const replaceMultiWhitespace = (str) => str.replace(MULTI_WHITESPACE_REGEX, ' '); 10 12 const transforms = { ··· 22 24 replaceMultiWhitespace, 23 25 lowercase 24 26 ]; 25 - export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; 27 + export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, removeNonAlphanumeric, replaceMultiWhitespace, transforms, strDefaultTransforms }; 26 28 //# sourceMappingURL=index.js.map
+1 -1
dist/esm/normalization/index.js.map
··· 1 - {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACnD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;AAErD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AAChF,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAC9D,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AAClH,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AACnG,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACjG,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;AAE9G,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAED,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAEF,OAAO,EACH,SAAS,EACT,IAAI,EACJ,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,sBAAsB,EACtB,UAAU,EACV,oBAAoB,EACvB,CAAA"} 1 + {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,0CAA0C,CAAC,CAAC;AACjF,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACxD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;AAErD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AAChF,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAC9D,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AAClH,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AACnG,MAAM,qBAAqB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;AAC5G,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACjG,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;AAE9G,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAED,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAEF,OAAO,EACH,SAAS,EACT,IAAI,EACJ,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,sBAAsB,EACtB,UAAU,EACV,oBAAoB,EACvB,CAAA"}