diff --git a/assets/charset3_result.hjson b/assets/charset3_result.hjson new file mode 100644 index 0000000..3ece395 --- /dev/null +++ b/assets/charset3_result.hjson @@ -0,0 +1,18 @@ +{ + a: šŸ˜€ + b: šŸ˜€ + c: Ć© + d: šŸ˜€x + e: a šŸ˜€ x + f: a Ć©šŸ˜€Ć© x + g: " šŸ‘‹šŸŒšŸš€" + h: " šŸ‘‹šŸŒšŸš€" + i: + ''' + šŸ‘‹ + šŸŒ + šŸš€ + ''' + j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 + k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 +} diff --git a/assets/charset3_result.json b/assets/charset3_result.json new file mode 100644 index 0000000..c9beb81 --- /dev/null +++ b/assets/charset3_result.json @@ -0,0 +1,13 @@ +{ + "a": "šŸ˜€", + "b": "šŸ˜€", + "c": "Ć©", + "d": "šŸ˜€x", + "e": "a šŸ˜€ x", + "f": "a Ć©šŸ˜€Ć© x", + "g": " šŸ‘‹šŸŒšŸš€", + "h": " šŸ‘‹šŸŒšŸš€", + "i": "šŸ‘‹\n šŸŒ\nšŸš€", + "j": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80", + "k": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80" +} diff --git a/assets/charset3_test.hjson b/assets/charset3_test.hjson new file mode 100644 index 0000000..f9af43c --- /dev/null +++ b/assets/charset3_test.hjson @@ -0,0 +1,15 @@ +a: šŸ˜€ +b: "\ud83d\ude00" +c: "\u00e9" +d: "\uD83D\uDE00x" +e: "a \uD83D\uDE00 x" +f: "a \u00e9\uD83D\uDE00\u00e9 x" +g: " \ud83d\udc4b\ud83c\udf0d\ud83d\ude80" +h: ' \ud83d\udc4b\ud83c\udf0d\ud83d\ude80' +i: ''' + šŸ‘‹ + šŸŒ + šŸš€ + ''' +j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 +k: '''\ud83d\udc4b\ud83c\udf0d\ud83d\ude80''' diff --git a/assets/charset4_result.hjson b/assets/charset4_result.hjson new file mode 100644 index 0000000..b41d3db --- /dev/null +++ b/assets/charset4_result.hjson @@ -0,0 +1 @@ +šŸ˜€ diff --git a/assets/charset4_result.json b/assets/charset4_result.json new file mode 100644 index 0000000..5dcf064 --- /dev/null +++ b/assets/charset4_result.json @@ -0,0 +1 @@ +"šŸ˜€" diff --git a/assets/charset4_test.hjson b/assets/charset4_test.hjson new file mode 100644 index 0000000..b41d3db --- /dev/null +++ b/assets/charset4_test.hjson @@ -0,0 +1 @@ +šŸ˜€ diff --git a/assets/comments2/charset3_result.hjson b/assets/comments2/charset3_result.hjson new file mode 100644 index 0000000..3ece395 --- /dev/null +++ b/assets/comments2/charset3_result.hjson @@ -0,0 +1,18 @@ +{ + a: šŸ˜€ + b: šŸ˜€ + c: Ć© + d: šŸ˜€x + e: a šŸ˜€ x + f: a Ć©šŸ˜€Ć© x + g: " šŸ‘‹šŸŒšŸš€" + h: " šŸ‘‹šŸŒšŸš€" + i: + ''' + šŸ‘‹ + šŸŒ + šŸš€ + ''' + j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 + k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 +} diff --git a/assets/comments2/charset4_result.hjson b/assets/comments2/charset4_result.hjson new file mode 100644 index 0000000..b41d3db --- /dev/null +++ b/assets/comments2/charset4_result.hjson @@ -0,0 +1 @@ +šŸ˜€ diff --git a/assets/comments3/charset3_result.hjson b/assets/comments3/charset3_result.hjson new file mode 100644 index 0000000..165106e --- /dev/null +++ b/assets/comments3/charset3_result.hjson @@ -0,0 +1,18 @@ +{ + a: šŸ˜€ + b: šŸ˜€ + c: Ć© + d: šŸ˜€x + e: a šŸ˜€ x + f: a Ć©šŸ˜€Ć© x + g: " šŸ‘‹šŸŒšŸš€" + h: " šŸ‘‹šŸŒšŸš€" + i: + ''' + šŸ‘‹ + šŸŒ + šŸš€ + ''' + j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 + k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 +} diff --git a/assets/comments3/charset4_result.hjson b/assets/comments3/charset4_result.hjson new file mode 100644 index 0000000..b41d3db --- /dev/null +++ b/assets/comments3/charset4_result.hjson @@ -0,0 +1 @@ +šŸ˜€ diff --git a/assets/failCharset2_test.hjson b/assets/failCharset2_test.hjson new file mode 100644 index 0000000..527593b --- /dev/null +++ b/assets/failCharset2_test.hjson @@ -0,0 +1,4 @@ +{ + # invalid surrogate pair + char: "\ud83d\uzzzz" +} diff --git a/assets/sorted/charset3_result.hjson b/assets/sorted/charset3_result.hjson new file mode 100644 index 0000000..3ece395 --- /dev/null +++ b/assets/sorted/charset3_result.hjson @@ -0,0 +1,18 @@ +{ + a: šŸ˜€ + b: šŸ˜€ + c: Ć© + d: šŸ˜€x + e: a šŸ˜€ x + f: a Ć©šŸ˜€Ć© x + g: " šŸ‘‹šŸŒšŸš€" + h: " šŸ‘‹šŸŒšŸš€" + i: + ''' + šŸ‘‹ + šŸŒ + šŸš€ + ''' + j: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 + k: \ud83d\udc4b\ud83c\udf0d\ud83d\ude80 +} diff --git a/assets/sorted/charset3_result.json b/assets/sorted/charset3_result.json new file mode 100644 index 0000000..c9beb81 --- /dev/null +++ b/assets/sorted/charset3_result.json @@ -0,0 +1,13 @@ +{ + "a": "šŸ˜€", + "b": "šŸ˜€", + "c": "Ć©", + "d": "šŸ˜€x", + "e": "a šŸ˜€ x", + "f": "a Ć©šŸ˜€Ć© x", + "g": " šŸ‘‹šŸŒšŸš€", + "h": " šŸ‘‹šŸŒšŸš€", + "i": "šŸ‘‹\n šŸŒ\nšŸš€", + "j": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80", + "k": "\\ud83d\\udc4b\\ud83c\\udf0d\\ud83d\\ude80" +} diff --git a/assets/sorted/charset4_result.hjson b/assets/sorted/charset4_result.hjson new file mode 100644 index 0000000..b41d3db --- /dev/null +++ b/assets/sorted/charset4_result.hjson @@ -0,0 +1 @@ +šŸ˜€ diff --git a/assets/sorted/charset4_result.json b/assets/sorted/charset4_result.json new file mode 100644 index 0000000..5dcf064 --- /dev/null +++ b/assets/sorted/charset4_result.json @@ -0,0 +1 @@ +"šŸ˜€" diff --git a/assets/testlist.txt b/assets/testlist.txt index 6c681fb..a70798c 100644 --- a/assets/testlist.txt +++ b/assets/testlist.txt @@ -1,4 +1,6 @@ charset2_test.hjson +charset3_test.hjson +charset4_test.hjson charset_test.hjson comments2_test.hjson comments3_test.hjson @@ -9,6 +11,7 @@ comments7_test.hjson comments_test.hjson empty_test.hjson failCharset1_test.hjson +failCharset2_test.hjson failJSON02_test.json failJSON05_test.json failJSON06_test.json diff --git a/decode.go b/decode.go index 98a7a2e..14c96c0 100644 --- a/decode.go +++ b/decode.go @@ -8,6 +8,8 @@ import ( "fmt" "reflect" "strings" + "unicode" + "unicode/utf16" ) const maxPointerDepth = 512 @@ -192,6 +194,30 @@ func unravelDestination(dest reflect.Value, t reflect.Type) (reflect.Value, refl return dest, t } +func hexValue(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'a' && c <= 'f': + return int(c - 'a' + 0xa) + case c >= 'A' && c <= 'F': + return int(c - 'A' + 0xa) + } + return -1 +} + +func (p *hjsonParser) peekHex4(offs int) rune { + r := rune(0) + for i := 0; i < 4; i++ { + hex := hexValue(p.peek(offs + i)) + if hex < 0 { + return -1 + } + r = r*16 + rune(hex) + } + return r +} + func (p *hjsonParser) readString(allowML bool) (string, error) { // Parse a string value. @@ -217,19 +243,22 @@ func (p *hjsonParser) readString(allowML bool) (string, error) { uffff := 0 for i := 0; i < 4; i++ { p.next() - var hex int - if p.ch >= '0' && p.ch <= '9' { - hex = int(p.ch - '0') - } else if p.ch >= 'a' && p.ch <= 'f' { - hex = int(p.ch - 'a' + 0xa) - } else if p.ch >= 'A' && p.ch <= 'F' { - hex = int(p.ch - 'A' + 0xa) - } else { + hex := hexValue(p.ch) + if hex < 0 { return "", p.errAt("Bad \\u char " + string(p.ch)) } uffff = uffff*16 + hex } - res.WriteRune(rune(uffff)) + r := rune(uffff) + if utf16.IsSurrogate(r) && p.peek(0) == '\\' && p.peek(1) == 'u' { + if r2 := utf16.DecodeRune(r, p.peekHex4(2)); r2 != unicode.ReplacementChar { + r = r2 + for i := 0; i < 6; i++ { + p.next() + } + } + } + res.WriteRune(r) } else if ech, ok := escapee[p.ch]; ok { res.WriteByte(ech) } else {