Decode \u surrogate pairs in quoted strings

assets/charset3_result.hjson

@@ -0,0 +1,18 @@
+{
+  a: 😀
+  b: 😀
+  c: é
+  d: 😀x
+  e: a  😀  x
+  f: a  é😀é  x
+  g: " 👋🌍🚀"
+  h: " 👋🌍🚀"
+  i:
+    '''
+    👋
+     🌍
+    🚀
+    '''
+  j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+  k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+}

assets/charset3_result.json

@@ -0,0 +1,13 @@
+{
+  "a": "😀",
+  "b": "😀",
+  "c": "é",
+  "d": "😀x",
+  "e": "a  😀  x",
+  "f": "a  é😀é  x",
+  "g": " 👋🌍🚀",
+  "h": " 👋🌍🚀",
+  "i": "👋\n 🌍\n🚀",
+  "j": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80",
+  "k": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80"
+}

assets/charset3_test.hjson

@@ -0,0 +1,15 @@
+a: 😀
+b: "\ud83d\ude00"
+c: "\u00e9"
+d: "\uD83D\uDE00x"
+e: "a  \uD83D\uDE00  x"
+f: "a  \u00e9\uD83D\uDE00\u00e9  x"
+g: " \ud83d\udc4b\ud83c\udf0d\ud83d\ude80"
+h: ' \ud83d\udc4b\ud83c\udf0d\ud83d\ude80'
+i: '''
+   👋
+    🌍
+   🚀
+   '''
+j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+k: '''\ud83d\udc4b\ud83c\udf0d\ud83d\ude80'''

assets/charset4_result.hjson

@@ -0,0 +1 @@
+😀

assets/charset4_result.json

@@ -0,0 +1 @@
+"😀"

assets/charset4_test.hjson

@@ -0,0 +1 @@
+😀

assets/comments2/charset3_result.hjson

@@ -0,0 +1,18 @@
+{
+  a: 😀
+  b: 😀
+  c: é
+  d: 😀x
+  e: a  😀  x
+  f: a  é😀é  x
+  g: " 👋🌍🚀"
+  h: " 👋🌍🚀"
+  i:
+    '''
+    👋
+     🌍
+    🚀
+    '''
+  j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+  k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+}

assets/comments2/charset4_result.hjson

@@ -0,0 +1 @@
+😀

assets/comments3/charset3_result.hjson

@@ -0,0 +1,18 @@
+{
+  a: 😀
+  b: 😀
+  c: é
+  d: 😀x
+  e: a  😀  x
+  f: a  é😀é  x
+  g: " 👋🌍🚀"
+  h: " 👋🌍🚀"
+  i: 
+    '''
+    👋
+     🌍
+    🚀
+    '''
+  j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+  k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+}

assets/comments3/charset4_result.hjson

@@ -0,0 +1 @@
+😀

assets/failCharset2_test.hjson

@@ -0,0 +1,4 @@
+{
+  # invalid surrogate pair
+  char: "\ud83d\uzzzz"
+}

assets/sorted/charset3_result.hjson

@@ -0,0 +1,18 @@
+{
+  a: 😀
+  b: 😀
+  c: é
+  d: 😀x
+  e: a  😀  x
+  f: a  é😀é  x
+  g: " 👋🌍🚀"
+  h: " 👋🌍🚀"
+  i:
+    '''
+    👋
+     🌍
+    🚀
+    '''
+  j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+  k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80
+}

assets/sorted/charset3_result.json

@@ -0,0 +1,13 @@
+{
+  "a": "😀",
+  "b": "😀",
+  "c": "é",
+  "d": "😀x",
+  "e": "a  😀  x",
+  "f": "a  é😀é  x",
+  "g": " 👋🌍🚀",
+  "h": " 👋🌍🚀",
+  "i": "👋\n 🌍\n🚀",
+  "j": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80",
+  "k": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80"
+}

assets/sorted/charset4_result.hjson

@@ -0,0 +1 @@
+😀

assets/sorted/charset4_result.json

@@ -0,0 +1 @@
+"😀"

assets/testlist.txt

@@ -1,4 +1,6 @@
 charset2_test.hjson
+charset3_test.hjson
+charset4_test.hjson
 charset_test.hjson
 comments2_test.hjson
 comments3_test.hjson
@@ -9,6 +11,7 @@ comments7_test.hjson
 comments_test.hjson
 empty_test.hjson
 failCharset1_test.hjson
+failCharset2_test.hjson
 failJSON02_test.json
 failJSON05_test.json
 failJSON06_test.json

decode.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"unicode"
+	"unicode/utf16"
 )
 
 const maxPointerDepth = 512
@@ -192,6 +194,30 @@ func unravelDestination(dest reflect.Value, t reflect.Type) (reflect.Value, refl
 	return dest, t
 }
 
+func hexValue(c byte) int {
+	switch {
+	case c >= '0' && c <= '9':
+		return int(c - '0')
+	case c >= 'a' && c <= 'f':
+		return int(c - 'a' + 0xa)
+	case c >= 'A' && c <= 'F':
+		return int(c - 'A' + 0xa)
+	}
+	return -1
+}
+
+func (p *hjsonParser) peekHex4(offs int) rune {
+	r := rune(0)
+	for i := 0; i < 4; i++ {
+		hex := hexValue(p.peek(offs + i))
+		if hex < 0 {
+			return -1
+		}
+		r = r*16 + rune(hex)
+	}
+	return r
+}
+
 func (p *hjsonParser) readString(allowML bool) (string, error) {
 
 	// Parse a string value.
@@ -217,19 +243,22 @@ func (p *hjsonParser) readString(allowML bool) (string, error) {
 				uffff := 0
 				for i := 0; i < 4; i++ {
 					p.next()
-					var hex int
-					if p.ch >= '0' && p.ch <= '9' {
-						hex = int(p.ch - '0')
-					} else if p.ch >= 'a' && p.ch <= 'f' {
-						hex = int(p.ch - 'a' + 0xa)
-					} else if p.ch >= 'A' && p.ch <= 'F' {
-						hex = int(p.ch - 'A' + 0xa)
-					} else {
+					hex := hexValue(p.ch)
+					if hex < 0 {
 						return "", p.errAt("Bad \\u char " + string(p.ch))
 					}
 					uffff = uffff*16 + hex
 				}
-				res.WriteRune(rune(uffff))
+				r := rune(uffff)
+				if utf16.IsSurrogate(r) && p.peek(0) == '\\' && p.peek(1) == 'u' {
+					if r2 := utf16.DecodeRune(r, p.peekHex4(2)); r2 != unicode.ReplacementChar {
+						r = r2
+						for i := 0; i < 6; i++ {
+							p.next()
+						}
+					}
+				}
+				res.WriteRune(r)
 			} else if ech, ok := escapee[p.ch]; ok {
 				res.WriteByte(ech)
 			} else {