handle more spaces in succession, handle new lines as a space

Author: Mario Hros

html2text.go

@@ -67,6 +67,13 @@ func HTMLEntitiesToText(htmlEntsText string) string {
 	return outBuf.String()
 }
 
+func writeSpace(outBuf *bytes.Buffer) {
+	bts := outBuf.Bytes()
+	if len(bts) > 0 && bts[len(bts)-1] != ' ' {
+		outBuf.WriteString(" ")
+	}
+}
+
 // HTML2Text converts html into a text form
 func HTML2Text(html string) string {
 	inLen := len(html)
@@ -87,7 +94,10 @@ func HTML2Text(html string) string {
 		}
 
 		switch {
-		case r <= 0xD, r == 0x85, r == 0x2028, r == 0x2029: // skip new lines
+		// skip new lines and spaces adding a single space if not there yet
+		case r <= 0xD, r == 0x85, r == 0x2028, r == 0x2029, // new lines
+			r == ' ', r >= 0x2008 && r <= 0x200B: // spaces
+			writeSpace(outBuf)
 			continue
 
 		case r == ';' && inEnt: // end of html entity

html2text_test.go

@@ -23,8 +23,10 @@ func TestHTML2Text(t *testing.T) {
 			So(HTML2Text(`some <div id="a" class="b">div</div>`), ShouldEqual, "some div")
 		})
 
-		Convey("Line breaks", func() {
+		Convey("Line breaks and spaces", func() {
+			So(HTML2Text("should    ignore more spaces"), ShouldEqual, "should ignore more spaces")
 			So(HTML2Text("should \nignore \r\nnew lines"), ShouldEqual, "should ignore new lines")
+			So(HTML2Text("a\nb\nc"), ShouldEqual, "a b c")
 			So(HTML2Text(`two<br>line<br/>breaks`), ShouldEqual, "two\r\nline\r\nbreaks")
 			So(HTML2Text(`<p>two</p><p>paragraphs</p>`), ShouldEqual, "two\r\n\r\nparagraphs")
 		})