From df1ce49053c29b18d6399002d2c4a8de3b2a4ecd Mon Sep 17 00:00:00 2001 From: Jeremy B Merrill Date: Wed, 2 Apr 2025 21:48:31 -0400 Subject: [PATCH 1/5] add trimSentences option to SentenceTokenizer.tokenize() --- lib/natural/tokenizers/sentence_tokenizer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/natural/tokenizers/sentence_tokenizer.js b/lib/natural/tokenizers/sentence_tokenizer.js index bf75cc375..0b9345d9c 100644 --- a/lib/natural/tokenizers/sentence_tokenizer.js +++ b/lib/natural/tokenizers/sentence_tokenizer.js @@ -144,7 +144,7 @@ class SentenceTokenizer extends Tokenizer { return originalText } - tokenize (text) { + tokenize (text, trimSentences=true) { this.replacementCounter = 0 this.replacementMap = new Map() this.delimiterMap = new Map() @@ -181,7 +181,7 @@ class SentenceTokenizer extends Tokenizer { const trimmedSentences = this.trim(newSentences) DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences)) - const trimmedSentences2 = trimmedSentences.map(sent => sent.trim()) + const trimmedSentences2 = trimmedSentences.map(sent => trimSentences? sent.trim() : trim ) DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2)) DEBUG && console.log('---End of sentence tokenization--------------------------') DEBUG && console.log('---Replacement map---------------------------------------') From 93acf3b5366cc2f5c263cf2f79929d55929aa08e Mon Sep 17 00:00:00 2001 From: Jeremy B Merrill Date: Thu, 3 Apr 2025 16:36:48 -0400 Subject: [PATCH 2/5] trimSentence argument is on the SentenceTokenizer constructor, not on tokenize() --- lib/natural/tokenizers/sentence_tokenizer.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/natural/tokenizers/sentence_tokenizer.js b/lib/natural/tokenizers/sentence_tokenizer.js index 0b9345d9c..467e87e87 100644 --- a/lib/natural/tokenizers/sentence_tokenizer.js +++ b/lib/natural/tokenizers/sentence_tokenizer.js @@ -41,13 +41,18 @@ function escapeRegExp (string) { } class SentenceTokenizer extends Tokenizer { - constructor (abbreviations) { + constructor (abbreviations, trimSentences) { super() if (abbreviations) { this.abbreviations = abbreviations } else { this.abbreviations = [] } + if (trimSentences === undefined) { + this.trimSentences = true + }else{ + this.trimSentences = trimSentences; + } this.replacementMap = null this.replacementCounter = 0 } @@ -144,7 +149,7 @@ class SentenceTokenizer extends Tokenizer { return originalText } - tokenize (text, trimSentences=true) { + tokenize (text) { this.replacementCounter = 0 this.replacementMap = new Map() this.delimiterMap = new Map() @@ -181,7 +186,7 @@ class SentenceTokenizer extends Tokenizer { const trimmedSentences = this.trim(newSentences) DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences)) - const trimmedSentences2 = trimmedSentences.map(sent => trimSentences? sent.trim() : trim ) + const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent ) DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2)) DEBUG && console.log('---End of sentence tokenization--------------------------') DEBUG && console.log('---Replacement map---------------------------------------') From 651fd45747a50964ca02cfb0470ceaf0bc47bd05 Mon Sep 17 00:00:00 2001 From: Jeremy B Merrill Date: Thu, 3 Apr 2025 16:36:56 -0400 Subject: [PATCH 3/5] add test for trimSentence --- spec/sentence_tokenizer_spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/spec/sentence_tokenizer_spec.ts b/spec/sentence_tokenizer_spec.ts index 8d1ba4c59..b149323d1 100644 --- a/spec/sentence_tokenizer_spec.ts +++ b/spec/sentence_tokenizer_spec.ts @@ -220,3 +220,16 @@ describe('sentence_tokenizer', function () { }) }) }) + +// describe('sentence_tokenizer with trimSentences set to false', function () { +// let tokenizer: Tokenizer + +// beforeAll(function () { +// tokenizer = new Tokenizer(['i.e.', 'etc.', 'vs.', 'Inc.', 'A.S.A.P.'], +// ['.', '!', '?', '\n', '\r', '...', '…'], false) +// }) + +// it('should tokenize strings but not trim whitespace if trimSentences is false', function () { +// expect(tokenizer.tokenize('This is a sentence. This is another sentence.')).toEqual(['This is a sentence. ', 'This is another sentence.']) +// }) +// }) From fbd1e1fd076f4196b75e842eade72dc57506b769 Mon Sep 17 00:00:00 2001 From: Hugo ter Doest Date: Wed, 21 May 2025 10:58:00 +0200 Subject: [PATCH 4/5] Update sentence_tokenizer.js --- lib/natural/tokenizers/sentence_tokenizer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/natural/tokenizers/sentence_tokenizer.js b/lib/natural/tokenizers/sentence_tokenizer.js index 467e87e87..a7be81e51 100644 --- a/lib/natural/tokenizers/sentence_tokenizer.js +++ b/lib/natural/tokenizers/sentence_tokenizer.js @@ -50,8 +50,8 @@ class SentenceTokenizer extends Tokenizer { } if (trimSentences === undefined) { this.trimSentences = true - }else{ - this.trimSentences = trimSentences; + } else { + this.trimSentences = trimSentences } this.replacementMap = null this.replacementCounter = 0 From 90e8e590fbb6c955a4a62a22bea72d701e79b984 Mon Sep 17 00:00:00 2001 From: Hugo ter Doest Date: Thu, 22 May 2025 15:24:16 +0200 Subject: [PATCH 5/5] Update sentence_tokenizer.js --- lib/natural/tokenizers/sentence_tokenizer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/natural/tokenizers/sentence_tokenizer.js b/lib/natural/tokenizers/sentence_tokenizer.js index a7be81e51..f27042c09 100644 --- a/lib/natural/tokenizers/sentence_tokenizer.js +++ b/lib/natural/tokenizers/sentence_tokenizer.js @@ -186,7 +186,7 @@ class SentenceTokenizer extends Tokenizer { const trimmedSentences = this.trim(newSentences) DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences)) - const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent ) + const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent) DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2)) DEBUG && console.log('---End of sentence tokenization--------------------------') DEBUG && console.log('---Replacement map---------------------------------------')