From 55f1c9bbc67fbb72b01c61ae47759067c8cb3ef8 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Thu, 2 May 2024 02:04:09 +0300 Subject: [PATCH 1/4] feat: add multi symbol sequences --- common/util.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/common/util.ts b/common/util.ts index 521b44144..a2723330a 100644 --- a/common/util.ts +++ b/common/util.ts @@ -135,12 +135,34 @@ export function neat(params: TemplateStringsArray, ...rest: string[]) { } const END_SYMBOLS = new Set(`."”;’'*!！?？)}]\`>~`.split('')) +const END_SEQUENCES = [ + '\n```', + '\n---', + '\n***', + '\n___', + '\n===', + '\n"""', +] const MID_SYMBOLS = new Set(`.)}’'!?\``.split('')) export function trimSentence(text: string) { let index = -1, checkpoint = -1 - for (let i = text.length - 1; i >= 0; i--) { + sentence_loop: for (let i = text.length - 1; i >= 0; i--) { + // first check for end sequences + for (const seq of END_SEQUENCES) { + if (text.slice(i, i + seq.length) === seq) { + // only trim if it's not an opening sequence + if ( + i + seq.length < text.length && + /\p{L}/u.test(text[i + seq.length]) + ) { + break; + } + index = i + seq.length - 1 + break sentence_loop + } + } if (END_SYMBOLS.has(text[i])) { // Skip ahead if the punctuation mark is preceded by white space if (i && /[\p{White_Space}\n<]/u.test(text[i - 1])) { From b864f8e3b43927e972e97198bac89a03a3a08385 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Thu, 2 May 2024 02:04:50 +0300 Subject: [PATCH 2/4] style: format with prettier --- common/util.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/common/util.ts b/common/util.ts index a2723330a..7227d4c2c 100644 --- a/common/util.ts +++ b/common/util.ts @@ -135,14 +135,7 @@ export function neat(params: TemplateStringsArray, ...rest: string[]) { } const END_SYMBOLS = new Set(`."”;’'*!！?？)}]\`>~`.split('')) -const END_SEQUENCES = [ - '\n```', - '\n---', - '\n***', - '\n___', - '\n===', - '\n"""', -] +const END_SEQUENCES = ['\n```', '\n---', '\n***', '\n___', '\n===', '\n"""'] const MID_SYMBOLS = new Set(`.)}’'!?\``.split('')) export function trimSentence(text: string) { @@ -153,11 +146,8 @@ export function trimSentence(text: string) { for (const seq of END_SEQUENCES) { if (text.slice(i, i + seq.length) === seq) { // only trim if it's not an opening sequence - if ( - i + seq.length < text.length && - /\p{L}/u.test(text[i + seq.length]) - ) { - break; + if (i + seq.length < text.length && /\p{L}/u.test(text[i + seq.length])) { + break } index = i + seq.length - 1 break sentence_loop From 9c28d18e596a8a1d7784ed46841d02ec5f83f7c6 Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Thu, 2 May 2024 02:05:08 +0300 Subject: [PATCH 3/4] test: add tests for multi-symbol sequences --- tests/trim-sentence.spec.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/trim-sentence.spec.ts b/tests/trim-sentence.spec.ts index 93d43bc4a..9d086cc86 100644 --- a/tests/trim-sentence.spec.ts +++ b/tests/trim-sentence.spec.ts @@ -106,4 +106,22 @@ describe('trimSentence', () => { const result = trimSentence(text) expect(result).to.eq(`Hello world.`) }) + + it('should stop at multi-symbol sequences', () => { + const text = '```javascript\nconst x = 1\n```\nMy code' + const result = trimSentence(text) + expect(result).to.eq('```javascript\nconst x = 1\n```') + }) + + it('should trim opening multi-symbol sequences that are followed by letters', () => { + const text = 'Beginning of my code block.\n```javascript' + const result = trimSentence(text) + expect(result).to.eq('Beginning of my code block.') + }) + + it('should trim duplicating symbols in multi-symbol sequences', () => { + const text = 'Hello World.\n------- Separated orphan' + const result = trimSentence(text) + expect(result).to.eq('Hello World.\n---') + }) }) From f7dcb2a125acca47c470a61850bd8ef6c63ead2f Mon Sep 17 00:00:00 2001 From: Thomas Mello Date: Tue, 14 May 2024 23:48:39 +0300 Subject: [PATCH 4/4] feat: add end comment sequence --- common/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/util.ts b/common/util.ts index 7227d4c2c..3362b9483 100644 --- a/common/util.ts +++ b/common/util.ts @@ -135,7 +135,7 @@ export function neat(params: TemplateStringsArray, ...rest: string[]) { } const END_SYMBOLS = new Set(`."”;’'*!！?？)}]\`>~`.split('')) -const END_SEQUENCES = ['\n```', '\n---', '\n***', '\n___', '\n===', '\n"""'] +const END_SEQUENCES = ['\n```', '\n---', '\n***', '\n___', '\n===', '\n"""', '\n*/'] const MID_SYMBOLS = new Set(`.)}’'!?\``.split('')) export function trimSentence(text: string) {