From 1b3877343531f7305d1c61050725d46fc87ac5c6 Mon Sep 17 00:00:00 2001 From: Tnze Date: Mon, 24 May 2021 22:25:44 +0800 Subject: [PATCH 01/23] first commit --- nbt/snbt_decode.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 nbt/snbt_decode.go diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go new file mode 100644 index 0000000..8d7ac89 --- /dev/null +++ b/nbt/snbt_decode.go @@ -0,0 +1,62 @@ +package nbt + +type token int + +const ( + ILLEGAL token = iota + + IDENT // name + + INT // 12345 + FLT // 12345.67 + + BYTE // b or B + SHORT // s or S + LONG // l or L + FLOAT // f or F + DOUBLE // d or D + + STRING // "abc" 'def' + + LPAREN // ( + LBRACK // [ + LBRACE // { + COMMA // , + PERIOD // . + + RPAREN // ) + RBRACK // ] + RBRACE // } + SEMICOLON // ; + COLON // : +) + +const ( + scanContinue = iota + scanError +) + +type scanner struct { + step func(c byte) int + parseState []int +} + +func (s *scanner) stateBeginValue(c byte) int { + switch c { + case '{': // beginning of TAG_Compound + s.step = s.stateCompound + case '[': // beginning of TAG_List + s.step = s.stateList + case '"', '\'': // beginning of TAG_String + + } + return scanError +} + +func (p *scanner) stateCompound(c byte) int { + return scanError +} + +func (p *scanner) stateList(c byte) int { + return scanError +} From 6846e10bb6c0945ce0dd89236dca677f0a57628d Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 00:47:57 +0800 Subject: [PATCH 02/23] Scan numbers --- nbt/snbt_decode.go | 139 ++++++++++++++++++++++++++++++++++++++- nbt/snbt_scanner_test.go | 23 +++++++ 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 nbt/snbt_scanner_test.go diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 8d7ac89..fe8a389 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -33,14 +33,60 @@ const ( const ( scanContinue = iota + scanSkipSpace + scanEndValue + scanEnd scanError ) +const maxNestingDepth = 10000 + type scanner struct { step func(c byte) int parseState []int } +// reset prepares the scanner for use. +// It must be called before calling s.step. +func (s *scanner) reset() { + s.step = s.stateBeginValue + s.parseState = s.parseState[0:0] +} + +// pushParseState pushes a new parse state p onto the parse stack. +// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned. +func (s *scanner) pushParseState(c byte, newParseState int, successState int) int { + s.parseState = append(s.parseState, newParseState) + if len(s.parseState) <= maxNestingDepth { + return successState + } + return scanError +} + +// popParseState pops a parse state (already obtained) off the stack +// and updates s.step accordingly. +func (s *scanner) popParseState() { + n := len(s.parseState) - 1 + s.parseState = s.parseState[0:n] + if n == 0 { + s.step = s.stateEndTop + //s.endTop = true + } else { + s.step = s.stateEndValue + } +} + +// stateEndTop is the state after finishing the top-level value, +// such as after reading `{}` or `[1,2,3]`. +// Only space characters should be seen now. +func (s *scanner) stateEndTop(c byte) int { + if !isSpace(c) { + // Complain about non-space byte on next call. + //s.error(c, "after top-level value") + } + return scanEnd +} + func (s *scanner) stateBeginValue(c byte) int { switch c { case '{': // beginning of TAG_Compound @@ -49,14 +95,103 @@ func (s *scanner) stateBeginValue(c byte) int { s.step = s.stateList case '"', '\'': // beginning of TAG_String + default: + if c >= '0' && c <= '9' { + s.step = s.stateNum1 + return scanContinue + } } return scanError } -func (p *scanner) stateCompound(c byte) int { +func (s *scanner) stateNum1(c byte) int { + if c >= '0' || c <= '9' { + s.step = s.stateNum1 + return scanContinue + } + if c == '.' { + s.step = s.stateNumDot + return scanContinue + } + return s.stateEndValue(c) +} + +// stateDot is the state after reading the integer and decimal point in a number, +// such as after reading `1.`. +func (s *scanner) stateNumDot(c byte) int { + if c >= '0' || c <= '9' { + s.step = s.stateNumDot0 + return scanContinue + } return scanError } -func (p *scanner) stateList(c byte) int { +// stateNumDot0 is the state after reading the integer, decimal point, and subsequent +// digits of a number, such as after reading `3.14`. +func (s *scanner) stateNumDot0(c byte) int { + if c >= '0' || c <= '9' { + s.step = s.stateNumDot0 + return scanContinue + } + return s.stateEndNumDotValue(c) +} + +func (s *scanner) stateCompound(c byte) int { + return s.error(c, "not implemented") +} + +func (s *scanner) stateList(c byte) int { + return s.error(c, "not implemented") +} + +func (s *scanner) stateEndNumValue(c byte) int { + if isSpace(c) { + s.step = s.stateEndValue + return scanSkipSpace + } + switch c { + case 'b', 'B': // TAG_Byte + s.step = s.stateEndValue + return scanSkipSpace + case 's', 'S': // TAG_Short + s.step = s.stateEndValue + return scanSkipSpace + case 'l', 'L': // TAG_Long + s.step = s.stateEndValue + return scanSkipSpace + case 'f', 'F', 'd', 'D': + return s.stateEndNumDotValue(c) + } + return s.stateEndValue(c) +} + +func (s *scanner) stateEndNumDotValue(c byte) int { + switch c { + case 'f', 'F': // TAG_Float + s.step = s.stateEndValue + return scanSkipSpace + case 'd', 'D': // TAG_Double + s.step = s.stateEndValue + return scanSkipSpace + } + return s.stateEndValue(c) +} + +func (s *scanner) stateEndValue(c byte) int { + return s.error(c, "not implemented") +} + +func (s *scanner) error(c byte, context string) int { + s.step = s.stateError return scanError } + +// stateError is the state after reaching a syntax error, +// such as after reading `[1}` or `5.1.2`. +func (s *scanner) stateError(c byte) int { + return scanError +} + +func isSpace(c byte) bool { + return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') +} diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go new file mode 100644 index 0000000..a012ec9 --- /dev/null +++ b/nbt/snbt_scanner_test.go @@ -0,0 +1,23 @@ +package nbt + +import "testing" + +func TestSNBT(t *testing.T) { + var s scanner + for _, str := range []string{ + "0", "1234567890", "3.1415926", + "255B", "1234s", "6666L", + "314F", "3.14f", "3.14159265358979323846264D", + } { + s.reset() + var scanCodes []int + for _, c := range []byte(str) { + res := s.step(c) + if res == scanError { + t.Errorf("scan error") + } + scanCodes = append(scanCodes, res) + } + t.Logf("scancodes: %v", scanCodes) + } +} From 0c0459d69de7cdb13e618cbc297edaccbe7ed878 Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 12:49:57 +0800 Subject: [PATCH 03/23] Scan compound --- nbt/snbt_decode.go | 212 ++++++++++++++++++++++++++++++++++----- nbt/snbt_scanner_test.go | 53 ++++++++-- 2 files changed, 231 insertions(+), 34 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index fe8a389..8b49fd8 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -1,5 +1,9 @@ package nbt +import ( + "errors" +) + type token int const ( @@ -32,18 +36,33 @@ const ( ) const ( - scanContinue = iota - scanSkipSpace + scanContinue = iota // uninteresting byte + scanBeginCompound // begin compound (after left-brace ) + scanCompoundTagName // just finished read tag name (before colon) + scanCompoundValue // just finished read value (before comma or right-brace ) + scanSkipSpace // space byte; can skip; known to be last "continue" result scanEndValue + scanEnd scanError ) +// These values are stored in the parseState stack. +// They give the current state of a composite value +// being scanned. If the parser is inside a nested value +// the parseState describes the nested state, outermost at entry 0. +const ( + parseCompoundName = iota // parsing tag name (before colon) + parseCompoundValue // parsing value (after colon) + parseListValue // parsing list +) + const maxNestingDepth = 10000 type scanner struct { step func(c byte) int parseState []int + err error } // reset prepares the scanner for use. @@ -82,30 +101,134 @@ func (s *scanner) popParseState() { func (s *scanner) stateEndTop(c byte) int { if !isSpace(c) { // Complain about non-space byte on next call. - //s.error(c, "after top-level value") + s.error(c, "after top-level value") } return scanEnd } func (s *scanner) stateBeginValue(c byte) int { + if isSpace(c) { + return scanSkipSpace + } switch c { case '{': // beginning of TAG_Compound - s.step = s.stateCompound + s.step = s.stateCompoundOrEmpty + return s.pushParseState(c, parseCompoundName, scanBeginCompound) case '[': // beginning of TAG_List s.step = s.stateList case '"', '\'': // beginning of TAG_String - + s.step = s.stateBeginString + return scanContinue + case '-': // beginning of negative number + s.step = s.stateNeg + return scanContinue default: - if c >= '0' && c <= '9' { + if isNumber(c) { s.step = s.stateNum1 return scanContinue } } - return scanError + return s.error(c, "looking for beginning of value") +} + +func (s *scanner) stateCompoundOrEmpty(c byte) int { + if isSpace(c) { + return scanSkipSpace + } + if c == '}' { + n := len(s.parseState) + s.parseState[n-1] = parseCompoundValue + return s.stateEndValue(c) + } + return s.stateBeginString(c) +} + +func (s *scanner) stateBeginString(c byte) int { + switch c { + case '\'': + s.step = s.stateInSqString + return scanContinue + case '"': + s.step = s.stateInDqString + return scanContinue + default: + if isNumOrLetter(c) { + s.step = s.stateInPureString + return scanContinue + } + } + return s.error(c, "looking for beginning of tag name string") +} + +func (s *scanner) stateInSqString(c byte) int { + if c == '\\' { + s.step = s.stateInSqStringEsc + return scanContinue + } + if c == '\'' { + s.step = s.stateEndValue + return scanContinue + } + if isNumOrLetter(c) { + return scanContinue + } + return s.stateEndValue(c) +} + +func (s *scanner) stateInSqStringEsc(c byte) int { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': + s.step = s.stateInSqString + return scanContinue + } + return s.error(c, "in string escape code") +} + +func (s *scanner) stateInDqString(c byte) int { + if c == '\\' { + s.step = s.stateInDqStringEsc + return scanContinue + } + if c == '"' { + s.step = s.stateEndValue + return scanContinue + } + if isNumOrLetter(c) { + return scanContinue + } + return s.stateEndValue(c) +} + +func (s *scanner) stateInDqStringEsc(c byte) int { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': + s.step = s.stateInDqString + return scanContinue + } + return s.error(c, "in string escape code") +} + +func (s *scanner) stateInPureString(c byte) int { + if isNumOrLetter(c) { + return scanContinue + } + return s.stateEndValue(c) +} + +func (s *scanner) stateList(c byte) int { + return s.error(c, "not implemented") +} + +func (s *scanner) stateNeg(c byte) int { + if !isNumber(c) { + s.error(c, "not a number after '-'") + } + s.step = s.stateNum1 + return scanContinue } func (s *scanner) stateNum1(c byte) int { - if c >= '0' || c <= '9' { + if isNumber(c) { s.step = s.stateNum1 return scanContinue } @@ -113,37 +236,29 @@ func (s *scanner) stateNum1(c byte) int { s.step = s.stateNumDot return scanContinue } - return s.stateEndValue(c) + return s.stateEndNumValue(c) } // stateDot is the state after reading the integer and decimal point in a number, // such as after reading `1.`. func (s *scanner) stateNumDot(c byte) int { - if c >= '0' || c <= '9' { + if isNumber(c) { s.step = s.stateNumDot0 return scanContinue } - return scanError + return s.error(c, "after decimal point in numeric literal") } // stateNumDot0 is the state after reading the integer, decimal point, and subsequent // digits of a number, such as after reading `3.14`. func (s *scanner) stateNumDot0(c byte) int { - if c >= '0' || c <= '9' { + if isNumber(c) { s.step = s.stateNumDot0 return scanContinue } return s.stateEndNumDotValue(c) } -func (s *scanner) stateCompound(c byte) int { - return s.error(c, "not implemented") -} - -func (s *scanner) stateList(c byte) int { - return s.error(c, "not implemented") -} - func (s *scanner) stateEndNumValue(c byte) int { if isSpace(c) { s.step = s.stateEndValue @@ -152,13 +267,13 @@ func (s *scanner) stateEndNumValue(c byte) int { switch c { case 'b', 'B': // TAG_Byte s.step = s.stateEndValue - return scanSkipSpace + return scanContinue case 's', 'S': // TAG_Short s.step = s.stateEndValue - return scanSkipSpace + return scanContinue case 'l', 'L': // TAG_Long s.step = s.stateEndValue - return scanSkipSpace + return scanContinue case 'f', 'F', 'd', 'D': return s.stateEndNumDotValue(c) } @@ -169,20 +284,51 @@ func (s *scanner) stateEndNumDotValue(c byte) int { switch c { case 'f', 'F': // TAG_Float s.step = s.stateEndValue - return scanSkipSpace + return scanContinue case 'd', 'D': // TAG_Double s.step = s.stateEndValue - return scanSkipSpace + return scanContinue } return s.stateEndValue(c) } func (s *scanner) stateEndValue(c byte) int { + n := len(s.parseState) + if n == 0 { + // Completed top-level before the current byte. + s.step = s.stateEndTop + //s.endTop = true + return s.stateEndTop(c) + } + if isSpace(c) { + s.step = s.stateEndValue + return scanSkipSpace + } + + ps := s.parseState[n-1] + switch ps { + case parseCompoundName: + if c == ':' { + s.parseState[n-1] = parseCompoundValue + s.step = s.stateBeginValue + return scanCompoundTagName + } + return s.error(c, "after object key") + case parseCompoundValue: + switch c { + case ',': + s.step = s.stateBeginString + return scanCompoundValue + case '}': + s.popParseState() + return scanEndValue + } + } return s.error(c, "not implemented") } - func (s *scanner) error(c byte, context string) int { s.step = s.stateError + s.err = errors.New(context) return scanError } @@ -195,3 +341,17 @@ func (s *scanner) stateError(c byte) int { func isSpace(c byte) bool { return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') } + +func isNumber(c byte) bool { + if c >= '0' && c <= '9' { + return true + } + return false +} + +func isNumOrLetter(c byte) bool { + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || isNumber(c) { + return true + } + return false +} diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index a012ec9..2f4e177 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -2,22 +2,59 @@ package nbt import "testing" -func TestSNBT(t *testing.T) { - var s scanner - for _, str := range []string{ +func TestSNBT_number(t *testing.T) { + goods := []string{ "0", "1234567890", "3.1415926", + "-0", "-1234567890", "-3.1415926", "255B", "1234s", "6666L", "314F", "3.14f", "3.14159265358979323846264D", - } { + } + bads := []string{ + ".0", "1234.5678.90", + "25-5B", "1234.s", + } + var s scanner + scan := func(str string) bool { s.reset() - var scanCodes []int for _, c := range []byte(str) { res := s.step(c) if res == scanError { - t.Errorf("scan error") + return false } - scanCodes = append(scanCodes, res) } - t.Logf("scancodes: %v", scanCodes) + return true + } + for _, str := range goods { + if scan(str) == false { + t.Errorf("scan valid data %q error: %v", str, s.err) + } + } + for _, str := range bads { + if scan(str) { + t.Errorf("scan invalid data %q success", str) + } + } +} + +func TestSNBT_compound(t *testing.T) { + goods := []string{ + `{}`, `{name:3.14f}`, `{ "name" : 12345 }`, + `{ abc: {}}`, + } + var s scanner + scan := func(str string) bool { + s.reset() + for _, c := range []byte(str) { + res := s.step(c) + if res == scanError { + return false + } + } + return true + } + for _, str := range goods { + if scan(str) == false { + t.Errorf("scan valid data %q error: %v", str, s.err) + } } } From 5320722ed04d3e67a4702ef98bf60f622436714e Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 14:25:08 +0800 Subject: [PATCH 04/23] Scan List and Array --- nbt/snbt_decode.go | 65 ++++++++++++++++++++++++++++++++-------- nbt/snbt_scanner_test.go | 27 ++++++++++++++++- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 8b49fd8..1a7fb8e 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -37,7 +37,10 @@ const ( const ( scanContinue = iota // uninteresting byte - scanBeginCompound // begin compound (after left-brace ) + scanBeginCompound // begin TAG_Compound (after left-brace ) + scanBeginList // begin TAG_List (after left-brack) + scanListValue // just finished read list value + scanListType // just finished read list type (after "B;" or "L;") scanCompoundTagName // just finished read tag name (before colon) scanCompoundValue // just finished read value (before comma or right-brace ) scanSkipSpace // space byte; can skip; known to be last "continue" result @@ -86,7 +89,7 @@ func (s *scanner) pushParseState(c byte, newParseState int, successState int) in // and updates s.step accordingly. func (s *scanner) popParseState() { n := len(s.parseState) - 1 - s.parseState = s.parseState[0:n] + s.parseState = s.parseState[:n] if n == 0 { s.step = s.stateEndTop //s.endTop = true @@ -108,6 +111,7 @@ func (s *scanner) stateEndTop(c byte) int { func (s *scanner) stateBeginValue(c byte) int { if isSpace(c) { + s.step = s.stateBeginValue return scanSkipSpace } switch c { @@ -115,17 +119,19 @@ func (s *scanner) stateBeginValue(c byte) int { s.step = s.stateCompoundOrEmpty return s.pushParseState(c, parseCompoundName, scanBeginCompound) case '[': // beginning of TAG_List - s.step = s.stateList + s.step = s.stateListOrArray + return s.pushParseState(c, parseListValue, scanBeginList) case '"', '\'': // beginning of TAG_String - s.step = s.stateBeginString - return scanContinue + return s.stateBeginString(c) case '-': // beginning of negative number s.step = s.stateNeg return scanContinue default: if isNumber(c) { - s.step = s.stateNum1 - return scanContinue + return s.stateNum1(c) + } + if isNumOrLetter(c) { + return s.stateBeginString(c) } } return s.error(c, "looking for beginning of value") @@ -144,6 +150,9 @@ func (s *scanner) stateCompoundOrEmpty(c byte) int { } func (s *scanner) stateBeginString(c byte) int { + if isSpace(c) { + return scanSkipSpace + } switch c { case '\'': s.step = s.stateInSqString @@ -157,7 +166,7 @@ func (s *scanner) stateBeginString(c byte) int { return scanContinue } } - return s.error(c, "looking for beginning of tag name string") + return s.error(c, "looking for beginning of string") } func (s *scanner) stateInSqString(c byte) int { @@ -215,8 +224,27 @@ func (s *scanner) stateInPureString(c byte) int { return s.stateEndValue(c) } -func (s *scanner) stateList(c byte) int { - return s.error(c, "not implemented") +func (s *scanner) stateListOrArray(c byte) int { + if isSpace(c) { + return scanSkipSpace + } + switch c { + case 'B', 'I', 'L': + s.step = s.stateListOrArrayT + return scanContinue + case ']': + return s.stateEndValue(c) + default: + return s.stateBeginValue(c) + } +} + +func (s *scanner) stateListOrArrayT(c byte) int { + if c == ';' { + s.step = s.stateBeginValue + return scanListType + } + return s.stateInPureString(c) } func (s *scanner) stateNeg(c byte) int { @@ -301,7 +329,6 @@ func (s *scanner) stateEndValue(c byte) int { return s.stateEndTop(c) } if isSpace(c) { - s.step = s.stateEndValue return scanSkipSpace } @@ -313,18 +340,30 @@ func (s *scanner) stateEndValue(c byte) int { s.step = s.stateBeginValue return scanCompoundTagName } - return s.error(c, "after object key") + return s.error(c, "after compound tag name") case parseCompoundValue: switch c { case ',': + s.parseState[n-1] = parseCompoundName s.step = s.stateBeginString return scanCompoundValue case '}': s.popParseState() return scanEndValue } + return s.error(c, "after compound value") + case parseListValue: + switch c { + case ',': + s.step = s.stateBeginValue + return scanListValue + case ']': + s.popParseState() + return scanEndValue + } + return s.error(c, "after list element") } - return s.error(c, "not implemented") + return s.error(c, "") } func (s *scanner) error(c byte, context string) int { s.step = s.stateError diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 2f4e177..44d1532 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -39,7 +39,32 @@ func TestSNBT_number(t *testing.T) { func TestSNBT_compound(t *testing.T) { goods := []string{ `{}`, `{name:3.14f}`, `{ "name" : 12345 }`, - `{ abc: {}}`, + `{ abc: { }}`, `{ "ab\"c": {}, def: 12345}`, + } + var s scanner + scan := func(str string) bool { + s.reset() + for _, c := range []byte(str) { + res := s.step(c) + if res == scanError { + return false + } + } + return true + } + for _, str := range goods { + if scan(str) == false { + t.Errorf("scan valid data %q error: %v", str, s.err) + } + } +} + +func TestSNBT_list(t *testing.T) { + goods := []string{ + `[]`, `[a, 'b', "c", d]`, // List of string + `[{}, {}, {"a\"b":520}]`, // List of Compound + `[B,C,D]`, `[L, "abc"]`, // List of string (like array) + `[B; 01B, 02B, 3B, 10B, 127B]`, // Array } var s scanner scan := func(str string) bool { From 69601cec2855a575213671f933b2becc1e0ea5bd Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 16:13:55 +0800 Subject: [PATCH 05/23] Fix Unquoted String scanning --- nbt/bigTest_test.snbt | 31 +++++++ nbt/{snbt_decode.go => snbt_scanner.go} | 109 ++++++++++-------------- nbt/snbt_scanner_test.go | 34 +++----- 3 files changed, 87 insertions(+), 87 deletions(-) create mode 100644 nbt/bigTest_test.snbt rename nbt/{snbt_decode.go => snbt_scanner.go} (85%) diff --git a/nbt/bigTest_test.snbt b/nbt/bigTest_test.snbt new file mode 100644 index 0000000..ec5e3b4 --- /dev/null +++ b/nbt/bigTest_test.snbt @@ -0,0 +1,31 @@ +{ + shortTest: 32767s, + longTest: 9223372036854775807L, + byteTest: 127b, + "byteArrayTest (the first 1000 values of (n*n*255+n*7)%100, starting with n=0 (0, 62, 34, 16, 8, ...))": [B; 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B], + "listTest (long)": [11L, 12L, 13L, 14L, 15L], + floatTest: 0.49823147f, + doubleTest: 0.4931287132182315d, + intTest: 2147483647, + "listTest (compound)": [ + { + created-on: 1264099775885L, + name: "Compound tag #0" + }, + { + created-on: 1264099775885L, + name: "Compound tag #1" + } + ], + "nested compound test": { + egg: { + name: "Eggbert", + value: 0.5f + }, + ham: { + name: "Hampus", + value: 0.75f + } + }, + stringTest: "HELLO WORLD THIS IS A TEST STRING ÅÄÖ!" +} diff --git a/nbt/snbt_decode.go b/nbt/snbt_scanner.go similarity index 85% rename from nbt/snbt_decode.go rename to nbt/snbt_scanner.go index 1a7fb8e..01dd23c 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_scanner.go @@ -4,37 +4,6 @@ import ( "errors" ) -type token int - -const ( - ILLEGAL token = iota - - IDENT // name - - INT // 12345 - FLT // 12345.67 - - BYTE // b or B - SHORT // s or S - LONG // l or L - FLOAT // f or F - DOUBLE // d or D - - STRING // "abc" 'def' - - LPAREN // ( - LBRACK // [ - LBRACE // { - COMMA // , - PERIOD // . - - RPAREN // ) - RBRACK // ] - RBRACE // } - SEMICOLON // ; - COLON // : -) - const ( scanContinue = iota // uninteresting byte scanBeginCompound // begin TAG_Compound (after left-brace ) @@ -130,7 +99,7 @@ func (s *scanner) stateBeginValue(c byte) int { if isNumber(c) { return s.stateNum1(c) } - if isNumOrLetter(c) { + if isAllowedInUnquotedString(c) { return s.stateBeginString(c) } } @@ -155,45 +124,42 @@ func (s *scanner) stateBeginString(c byte) int { } switch c { case '\'': - s.step = s.stateInSqString + s.step = s.stateInSingleQuotedString return scanContinue case '"': - s.step = s.stateInDqString + s.step = s.stateInDoubleQuotedString return scanContinue default: - if isNumOrLetter(c) { - s.step = s.stateInPureString + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString return scanContinue } } return s.error(c, "looking for beginning of string") } -func (s *scanner) stateInSqString(c byte) int { +func (s *scanner) stateInSingleQuotedString(c byte) int { if c == '\\' { - s.step = s.stateInSqStringEsc + s.step = s.stateInSingleQuotedStringEsc return scanContinue } if c == '\'' { s.step = s.stateEndValue return scanContinue } - if isNumOrLetter(c) { - return scanContinue - } - return s.stateEndValue(c) + return scanContinue } -func (s *scanner) stateInSqStringEsc(c byte) int { +func (s *scanner) stateInSingleQuotedStringEsc(c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': - s.step = s.stateInSqString + s.step = s.stateInSingleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInDqString(c byte) int { +func (s *scanner) stateInDoubleQuotedString(c byte) int { if c == '\\' { s.step = s.stateInDqStringEsc return scanContinue @@ -202,23 +168,20 @@ func (s *scanner) stateInDqString(c byte) int { s.step = s.stateEndValue return scanContinue } - if isNumOrLetter(c) { - return scanContinue - } - return s.stateEndValue(c) + return scanContinue } func (s *scanner) stateInDqStringEsc(c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': - s.step = s.stateInDqString + s.step = s.stateInDoubleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInPureString(c byte) int { - if isNumOrLetter(c) { +func (s *scanner) stateInUnquotedString(c byte) int { + if isAllowedInUnquotedString(c) { return scanContinue } return s.stateEndValue(c) @@ -244,15 +207,19 @@ func (s *scanner) stateListOrArrayT(c byte) int { s.step = s.stateBeginValue return scanListType } - return s.stateInPureString(c) + return s.stateInUnquotedString(c) } func (s *scanner) stateNeg(c byte) int { - if !isNumber(c) { - s.error(c, "not a number after '-'") + if isNumber(c) { + s.step = s.stateNum1 + return scanContinue } - s.step = s.stateNum1 - return scanContinue + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanContinue + } + return s.error(c, "not a number after '-'") } func (s *scanner) stateNum1(c byte) int { @@ -264,6 +231,10 @@ func (s *scanner) stateNum1(c byte) int { s.step = s.stateNumDot return scanContinue } + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanContinue + } return s.stateEndNumValue(c) } @@ -274,6 +245,10 @@ func (s *scanner) stateNumDot(c byte) int { s.step = s.stateNumDot0 return scanContinue } + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanContinue + } return s.error(c, "after decimal point in numeric literal") } @@ -284,6 +259,10 @@ func (s *scanner) stateNumDot0(c byte) int { s.step = s.stateNumDot0 return scanContinue } + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanContinue + } return s.stateEndNumDotValue(c) } @@ -382,15 +361,13 @@ func isSpace(c byte) bool { } func isNumber(c byte) bool { - if c >= '0' && c <= '9' { - return true - } - return false + return c >= '0' && c <= '9' } -func isNumOrLetter(c byte) bool { - if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || isNumber(c) { - return true - } - return false +func isAllowedInUnquotedString(c byte) bool { + return c == '_' || c == '-' || + c == '.' || c == '+' || + c >= '0' && c <= '9' || + c >= 'A' && c <= 'Z' || + c >= 'a' && c <= 'z' } diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 44d1532..4af3689 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -1,6 +1,9 @@ package nbt -import "testing" +import ( + _ "embed" + "testing" +) func TestSNBT_number(t *testing.T) { goods := []string{ @@ -9,10 +12,6 @@ func TestSNBT_number(t *testing.T) { "255B", "1234s", "6666L", "314F", "3.14f", "3.14159265358979323846264D", } - bads := []string{ - ".0", "1234.5678.90", - "25-5B", "1234.s", - } var s scanner scan := func(str string) bool { s.reset() @@ -29,36 +28,29 @@ func TestSNBT_number(t *testing.T) { t.Errorf("scan valid data %q error: %v", str, s.err) } } - for _, str := range bads { - if scan(str) { - t.Errorf("scan invalid data %q success", str) - } - } } +//go:embed bigTest_test.snbt +var bigTest string + func TestSNBT_compound(t *testing.T) { goods := []string{ `{}`, `{name:3.14f}`, `{ "name" : 12345 }`, - `{ abc: { }}`, `{ "ab\"c": {}, def: 12345}`, + `{ abc: { }}`, `{ "a b\"c": {}, def: 12345}`, + bigTest, } var s scanner - scan := func(str string) bool { + for _, str := range goods { s.reset() - for _, c := range []byte(str) { + for i, c := range []byte(str) { res := s.step(c) if res == scanError { - return false + t.Errorf("scan valid data %q error: %v at [%d]", str[:i], s.err, i) + break } } - return true - } - for _, str := range goods { - if scan(str) == false { - t.Errorf("scan valid data %q error: %v", str, s.err) - } } } - func TestSNBT_list(t *testing.T) { goods := []string{ `[]`, `[a, 'b', "c", d]`, // List of string From c44af2bf342077dd076f69c0b2b87b975ff78d20 Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 20:10:57 +0800 Subject: [PATCH 06/23] NBT convert part 1 --- nbt/snbt_decode.go | 92 ++++++++++++++++++++++++++++++++++++++++ nbt/snbt_decode_test.go | 16 +++++++ nbt/snbt_scanner.go | 71 ++++++++++++++++++++++++++++--- nbt/snbt_scanner_test.go | 9 ++++ 4 files changed, 181 insertions(+), 7 deletions(-) create mode 100644 nbt/snbt_decode.go create mode 100644 nbt/snbt_decode_test.go diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go new file mode 100644 index 0000000..4971071 --- /dev/null +++ b/nbt/snbt_decode.go @@ -0,0 +1,92 @@ +package nbt + +import ( + "fmt" +) + +type decodeState struct { + data []byte + off int // next read offset in data + opcode int // last read result + scan scanner +} + +const phasePanicMsg = "SNBT decoder out of sync - data changing underfoot?" + +func (e *Encoder) WriteSNBT(snbt string) error { + d := decodeState{data: []byte(snbt)} + d.scan.reset() + d.scanWhile(scanSkipSpace) + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginLiteral: + + case scanBeginCompound: + panic("not implemented") + + case scanBeginList: + panic("not implemented") + } + return nil +} + +func writeLiteral(e *Encoder, d *decodeState) error { + start := d.readIndex() + d.scanNext() + d.scanWhile(scanContinue) + literal := d.data[start:d.readIndex()] + fmt.Printf("%d %d [%d]- %q\n", start, d.off, d.opcode, literal) + + switch literal[0] { + case '"', '\'': // TAG_String + str := literal // TODO: Parse string + e.writeTag(TagString, "") + e.writeInt16(int16(len(str))) + e.w.Write(str) + + default: + e.w.Write(literal) // TODO: Parse other literal + } + return nil +} + +func writeCompound(e *Encoder, d *decodeState) error { + e.writeTag(TagCompound, "") + return nil +} + +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + if d.off < len(d.data) { + d.opcode = d.scan.step(d.data[d.off]) + d.off++ + } else { + //d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(data) { + newOp := s.step(data[i]) + i++ + if newOp != op { + d.opcode = newOp + d.off = i + return + } + } + + d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go new file mode 100644 index 0000000..144a418 --- /dev/null +++ b/nbt/snbt_decode_test.go @@ -0,0 +1,16 @@ +package nbt + +import ( + "bytes" + "testing" +) + +func TestEncoder_WriteSNBT(t *testing.T) { + var buf bytes.Buffer + e := NewEncoder(&buf) + if err := e.WriteSNBT(`"12345"`); err != nil { + t.Fatal(err) + } + t.Log(buf.Bytes()) + +} diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 01dd23c..5e26e8d 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -2,10 +2,12 @@ package nbt import ( "errors" + "sync" ) const ( scanContinue = iota // uninteresting byte + scanBeginLiteral // end implied by next result != scanContinue scanBeginCompound // begin TAG_Compound (after left-brace ) scanBeginList // begin TAG_List (after left-brack) scanListValue // just finished read list value @@ -35,6 +37,7 @@ type scanner struct { step func(c byte) int parseState []int err error + endTop bool } // reset prepares the scanner for use. @@ -44,6 +47,28 @@ func (s *scanner) reset() { s.parseState = s.parseState[0:0] } +var scannerPool = sync.Pool{ + New: func() interface{} { + return &scanner{} + }, +} + +func newScanner() *scanner { + scan := scannerPool.Get().(*scanner) + // scan.reset by design doesn't set bytes to zero + //scan.bytes = 0 + scan.reset() + return scan +} + +func freeScanner(scan *scanner) { + // Avoid hanging on to too much memory in extreme cases. + if len(scan.parseState) > 1024 { + scan.parseState = nil + } + scannerPool.Put(scan) +} + // pushParseState pushes a new parse state p onto the parse stack. // an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned. func (s *scanner) pushParseState(c byte, newParseState int, successState int) int { @@ -67,6 +92,25 @@ func (s *scanner) popParseState() { } } +// eof tells the scanner that the end of input has been reached. +// It returns a scan status just as s.step does. +func (s *scanner) eof() int { + if s.err != nil { + return scanError + } + if s.endTop { + return scanEnd + } + s.step(' ') + if s.endTop { + return scanEnd + } + if s.err == nil { + s.err = errors.New("unexpected end of JSON input") + } + return scanError +} + // stateEndTop is the state after finishing the top-level value, // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. @@ -94,10 +138,11 @@ func (s *scanner) stateBeginValue(c byte) int { return s.stateBeginString(c) case '-': // beginning of negative number s.step = s.stateNeg - return scanContinue + return scanBeginLiteral default: if isNumber(c) { - return s.stateNum1(c) + s.stateNum0(c) + return scanBeginLiteral } if isAllowedInUnquotedString(c) { return s.stateBeginString(c) @@ -125,14 +170,14 @@ func (s *scanner) stateBeginString(c byte) int { switch c { case '\'': s.step = s.stateInSingleQuotedString - return scanContinue + return scanBeginLiteral case '"': s.step = s.stateInDoubleQuotedString - return scanContinue + return scanBeginLiteral default: if isAllowedInUnquotedString(c) { s.step = s.stateInUnquotedString - return scanContinue + return scanBeginLiteral } } return s.error(c, "looking for beginning of string") @@ -211,6 +256,18 @@ func (s *scanner) stateListOrArrayT(c byte) int { } func (s *scanner) stateNeg(c byte) int { + if isNumber(c) { + s.step = s.stateNum0 + return scanBeginLiteral + } + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanBeginLiteral + } + return s.error(c, "not a number after '-'") +} + +func (s *scanner) stateNum0(c byte) int { if isNumber(c) { s.step = s.stateNum1 return scanContinue @@ -219,7 +276,7 @@ func (s *scanner) stateNeg(c byte) int { s.step = s.stateInUnquotedString return scanContinue } - return s.error(c, "not a number after '-'") + return s.stateEndNumValue(c) } func (s *scanner) stateNum1(c byte) int { @@ -304,7 +361,7 @@ func (s *scanner) stateEndValue(c byte) int { if n == 0 { // Completed top-level before the current byte. s.step = s.stateEndTop - //s.endTop = true + s.endTop = true return s.stateEndTop(c) } if isSpace(c) { diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 4af3689..7cc09aa 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -5,6 +5,15 @@ import ( "testing" ) +func TestSNBT_checkScanCode(t *testing.T) { + t.SkipNow() + var s scanner + s.reset() + for _, c := range []byte(`{ "a b\"c": {}, def: 12345}`) { + t.Logf("[%c] - %d", c, s.step(c)) + } +} + func TestSNBT_number(t *testing.T) { goods := []string{ "0", "1234567890", "3.1415926", From 1f7d07b2798cf1d22fa6db8199192e1eda311d1e Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 25 May 2021 22:03:14 +0800 Subject: [PATCH 07/23] fix finish number error --- nbt/snbt_decode.go | 2 +- nbt/snbt_scanner.go | 6 +----- nbt/snbt_scanner_test.go | 5 +++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 4971071..1185864 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -22,7 +22,7 @@ func (e *Encoder) WriteSNBT(snbt string) error { panic(phasePanicMsg) case scanBeginLiteral: - + return writeLiteral(e, &d) case scanBeginCompound: panic("not implemented") diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 5e26e8d..e47ddf9 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -86,7 +86,7 @@ func (s *scanner) popParseState() { s.parseState = s.parseState[:n] if n == 0 { s.step = s.stateEndTop - //s.endTop = true + s.endTop = true } else { s.step = s.stateEndValue } @@ -324,10 +324,6 @@ func (s *scanner) stateNumDot0(c byte) int { } func (s *scanner) stateEndNumValue(c byte) int { - if isSpace(c) { - s.step = s.stateEndValue - return scanSkipSpace - } switch c { case 'b', 'B': // TAG_Byte s.step = s.stateEndValue diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 7cc09aa..029288a 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -6,12 +6,13 @@ import ( ) func TestSNBT_checkScanCode(t *testing.T) { - t.SkipNow() + //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`{ "a b\"c": {}, def: 12345}`) { + for _, c := range []byte(`1234`) { t.Logf("[%c] - %d", c, s.step(c)) } + t.Logf("[%c] - %d", ' ', s.eof()) } func TestSNBT_number(t *testing.T) { From 00fcdb4b14391279749a6778e17d10372d36ba29 Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 26 May 2021 01:35:58 +0800 Subject: [PATCH 08/23] generate NBT TAG_Compound --- nbt/snbt_decode.go | 57 ++++++++++++++++++++++++++++++---------- nbt/snbt_decode_test.go | 2 +- nbt/snbt_scanner.go | 1 + nbt/snbt_scanner_test.go | 4 +-- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 1185864..9495812 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -1,9 +1,5 @@ package nbt -import ( - "fmt" -) - type decodeState struct { data []byte off int // next read offset in data @@ -16,28 +12,27 @@ const phasePanicMsg = "SNBT decoder out of sync - data changing underfoot?" func (e *Encoder) WriteSNBT(snbt string) error { d := decodeState{data: []byte(snbt)} d.scan.reset() + return writeValue(e, &d, "") +} + +func writeValue(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanSkipSpace) switch d.opcode { default: panic(phasePanicMsg) - case scanBeginLiteral: - return writeLiteral(e, &d) + return writeLiteral(e, d, tagName) case scanBeginCompound: - panic("not implemented") - + return writeCompound(e, d, tagName) case scanBeginList: panic("not implemented") } - return nil } -func writeLiteral(e *Encoder, d *decodeState) error { +func writeLiteral(e *Encoder, d *decodeState, tagName string) error { start := d.readIndex() - d.scanNext() d.scanWhile(scanContinue) literal := d.data[start:d.readIndex()] - fmt.Printf("%d %d [%d]- %q\n", start, d.off, d.opcode, literal) switch literal[0] { case '"', '\'': // TAG_String @@ -52,8 +47,42 @@ func writeLiteral(e *Encoder, d *decodeState) error { return nil } -func writeCompound(e *Encoder, d *decodeState) error { - e.writeTag(TagCompound, "") +func writeCompound(e *Encoder, d *decodeState, tagName string) error { + e.writeTag(TagCompound, tagName) + for { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndValue { + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + // read tag name + start := d.readIndex() + d.scanWhile(scanContinue) + tagName := string(d.data[start:d.readIndex()]) + // read value + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanCompoundTagName { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + writeLiteral(e, d, tagName) + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndValue { + break + } + if d.opcode != scanCompoundValue { + panic(phasePanicMsg) + } + } + e.w.Write([]byte{TagEnd}) return nil } diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 144a418..f6a43ec 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -8,7 +8,7 @@ import ( func TestEncoder_WriteSNBT(t *testing.T) { var buf bytes.Buffer e := NewEncoder(&buf) - if err := e.WriteSNBT(`"12345"`); err != nil { + if err := e.WriteSNBT(`{ abc: a123}`); err != nil { t.Fatal(err) } t.Log(buf.Bytes()) diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index e47ddf9..49cc566 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -397,6 +397,7 @@ func (s *scanner) stateEndValue(c byte) int { } return s.error(c, "") } + func (s *scanner) error(c byte, context string) int { s.step = s.stateError s.err = errors.New(context) diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 029288a..83b4c9d 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -6,10 +6,10 @@ import ( ) func TestSNBT_checkScanCode(t *testing.T) { - //t.SkipNow() + t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`1234`) { + for _, c := range []byte(`{ "a b\"c": {}, def: 12345, 'gh"i': 0.123f}`) { t.Logf("[%c] - %d", c, s.step(c)) } t.Logf("[%c] - %d", ' ', s.eof()) From 3b2f81ea1f5619b99b6aafcfe565cf9814adc09d Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 26 May 2021 02:15:45 +0800 Subject: [PATCH 09/23] Quoted string --- nbt/snbt_decode.go | 42 ++++++++++++++++++++++++++++++++--------- nbt/snbt_decode_test.go | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 9495812..1229557 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -1,5 +1,9 @@ package nbt +import ( + "strings" +) + type decodeState struct { data []byte off int // next read offset in data @@ -33,16 +37,12 @@ func writeLiteral(e *Encoder, d *decodeState, tagName string) error { start := d.readIndex() d.scanWhile(scanContinue) literal := d.data[start:d.readIndex()] - - switch literal[0] { - case '"', '\'': // TAG_String - str := literal // TODO: Parse string - e.writeTag(TagString, "") + switch v := parseLiteral(literal); v.(type) { + case string: + str := v.(string) + e.writeTag(TagString, tagName) e.writeInt16(int16(len(str))) - e.w.Write(str) - - default: - e.w.Write(literal) // TODO: Parse other literal + e.w.Write([]byte(str)) } return nil } @@ -119,3 +119,27 @@ func (d *decodeState) scanWhile(op int) { d.off = len(data) + 1 // mark processed EOF with len+1 d.opcode = d.scan.eof() } + +// parseLiteral parse an SNBT literal, might be +// TAG_String, TAG_Int, TAG_Float, ... etc. +// so returned value is one of string, int32, float32 ... +func parseLiteral(literal []byte) interface{} { + switch literal[0] { + case '"', '\'': // Quoted String + var sb strings.Builder + for i := 1; ; i++ { + c := literal[i] + switch c { + case literal[0]: + return sb.String() + case '\\': + i++ + c = literal[i] + } + sb.WriteByte(c) + } + default: + + } + panic(phasePanicMsg) +} diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index f6a43ec..023bbea 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -8,7 +8,7 @@ import ( func TestEncoder_WriteSNBT(t *testing.T) { var buf bytes.Buffer e := NewEncoder(&buf) - if err := e.WriteSNBT(`{ abc: a123}`); err != nil { + if err := e.WriteSNBT(`{ "abc": "a123"}`); err != nil { t.Fatal(err) } t.Log(buf.Bytes()) From e2fbef9beaef62a8393e1b411a6789a88ab34b62 Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 26 May 2021 03:36:12 +0800 Subject: [PATCH 10/23] Support literal and compound --- nbt/snbt_decode.go | 98 ++++++++++++++++++++++++++++++++++++++++- nbt/snbt_decode_test.go | 2 +- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 1229557..418e391 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -1,6 +1,8 @@ package nbt import ( + "math" + "strconv" "strings" ) @@ -43,6 +45,24 @@ func writeLiteral(e *Encoder, d *decodeState, tagName string) error { e.writeTag(TagString, tagName) e.writeInt16(int16(len(str))) e.w.Write([]byte(str)) + case int8: + e.writeTag(TagByte, tagName) + e.w.Write([]byte{byte(v.(int8))}) + case int16: + e.writeTag(TagShort, tagName) + e.writeInt16(v.(int16)) + case int32: + e.writeTag(TagInt, tagName) + e.writeInt32(v.(int32)) + case int64: + e.writeTag(TagLong, tagName) + e.writeInt64(v.(int64)) + case float32: + e.writeTag(TagFloat, tagName) + e.writeInt32(int32(math.Float32bits(v.(float32)))) + case float64: + e.writeTag(TagDouble, tagName) + e.writeInt64(int64(math.Float64bits(v.(float64)))) } return nil } @@ -68,8 +88,7 @@ func writeCompound(e *Encoder, d *decodeState, tagName string) error { if d.opcode != scanCompoundTagName { panic(phasePanicMsg) } - d.scanWhile(scanSkipSpace) - writeLiteral(e, d, tagName) + writeValue(e, d, tagName) // Next token must be , or }. if d.opcode == scanSkipSpace { @@ -127,6 +146,7 @@ func parseLiteral(literal []byte) interface{} { switch literal[0] { case '"', '\'': // Quoted String var sb strings.Builder + sb.Grow(len(literal) - 2) for i := 1; ; i++ { c := literal[i] switch c { @@ -139,7 +159,81 @@ func parseLiteral(literal []byte) interface{} { sb.WriteByte(c) } default: + strlen := len(literal) + integer := true + number := true + unqstr := true + var numberType byte + for i, c := range literal { + if isNumber(c) { + continue + } else if integer { + if i == strlen-1 && isIntegerType(c) { + numberType = c + strlen-- + } else if i > 0 || i == 0 && c != '-' { + integer = false + if i == 0 || c != '.' { + number = false + } + } + } else if number { + if i == strlen-1 && isFloatType(c) { + numberType = c + } else { + number = false + } + } else if !isAllowedInUnquotedString(c) { + unqstr = false + } + } + if integer { + num, err := strconv.ParseInt(string(literal[:strlen]), 10, 64) + if err != nil { + panic(err) + } + switch numberType { + case 'B', 'b': + return int8(num) + case 'S', 's': + return int16(num) + default: + return int32(num) + case 'L', 'l': + return num + case 'F', 'f': + return float32(num) + case 'D', 'd': + return float64(num) + } + } else if number { + num, err := strconv.ParseFloat(string(literal[:strlen-1]), 64) + if err != nil { + panic(err) + } + switch numberType { + case 'F', 'f': + return float32(num) + case 'D', 'd': + fallthrough + default: + return num + } + } else if unqstr { + return string(literal) + } } panic(phasePanicMsg) } + +func isIntegerType(c byte) bool { + return isFloatType(c) || + c == 'B' || c == 'b' || + c == 's' || c == 'S' || + c == 'L' || c == 'l' +} + +func isFloatType(c byte) bool { + return c == 'F' || c == 'f' || c == 'D' || c == 'd' +} diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 023bbea..2053d80 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -8,7 +8,7 @@ import ( func TestEncoder_WriteSNBT(t *testing.T) { var buf bytes.Buffer e := NewEncoder(&buf) - if err := e.WriteSNBT(`{ "abc": "a123"}`); err != nil { + if err := e.WriteSNBT(`{ abc: 123F, def: {}}`); err != nil { t.Fatal(err) } t.Log(buf.Bytes()) From 233b44a13f675bc201cf15964aa2e2cfe2c18a2f Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 26 May 2021 15:02:49 +0800 Subject: [PATCH 11/23] Basic SNBT list support --- nbt/snbt_decode.go | 50 +++++++++++++++++++++++++++++++++++++++- nbt/snbt_decode_test.go | 2 +- nbt/snbt_scanner.go | 2 +- nbt/snbt_scanner_test.go | 4 ++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 418e391..f450ff8 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -1,6 +1,7 @@ package nbt import ( + "bytes" "math" "strconv" "strings" @@ -31,7 +32,7 @@ func writeValue(e *Encoder, d *decodeState, tagName string) error { case scanBeginCompound: return writeCompound(e, d, tagName) case scanBeginList: - panic("not implemented") + return writeListOrArray(e, d, tagName) } } @@ -105,6 +106,53 @@ func writeCompound(e *Encoder, d *decodeState, tagName string) error { return nil } +func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndValue { // ']', empty TAG_List + e.writeTag(TagList, tagName) + e.writeInt32(0) + return nil + } + + start := d.readIndex() + + switch d.opcode { + case scanBeginLiteral: + d.scanWhile(scanContinue) + literal := d.data[start:d.readIndex()] + if d.opcode == scanListType { // TAG_X_Array + //listType := literal[0] + break + } + if d.opcode != scanListValue { // TAG_List + panic(phasePanicMsg) + } + // TODO + parseLiteral(literal) + fallthrough + case scanBeginList: // TAG_List + // We don't know the length of the List, + // so we read them into a buffer and count. + var buf bytes.Buffer + e2 := NewEncoder(&buf) + var count int + for { + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndValue { + break + } + if d.opcode == scanListValue { + // TODO + e2.writeInt32(0) + } + } + e.writeListHeader(TagList, tagName, count) + e.w.Write(buf.Bytes()) + case scanBeginCompound: // TAG_List + } + return nil +} + // readIndex returns the position of the last byte read. func (d *decodeState) readIndex() int { return d.off - 1 diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 2053d80..54e9e54 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -8,7 +8,7 @@ import ( func TestEncoder_WriteSNBT(t *testing.T) { var buf bytes.Buffer e := NewEncoder(&buf) - if err := e.WriteSNBT(`{ abc: 123F, def: {}}`); err != nil { + if err := e.WriteSNBT(`[1,2,3]`); err != nil { t.Fatal(err) } t.Log(buf.Bytes()) diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 49cc566..d97e8d2 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -239,7 +239,7 @@ func (s *scanner) stateListOrArray(c byte) int { switch c { case 'B', 'I', 'L': s.step = s.stateListOrArrayT - return scanContinue + return scanBeginLiteral case ']': return s.stateEndValue(c) default: diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 83b4c9d..5810724 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -6,10 +6,10 @@ import ( ) func TestSNBT_checkScanCode(t *testing.T) { - t.SkipNow() + //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`{ "a b\"c": {}, def: 12345, 'gh"i': 0.123f}`) { + for _, c := range []byte(`[I;1,2,3]`) { t.Logf("[%c] - %d", c, s.step(c)) } t.Logf("[%c] - %d", ' ', s.eof()) From fa2ab9a5d576d98c26ce1021273dc2baa04426c1 Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 26 May 2021 21:43:05 +0800 Subject: [PATCH 12/23] make tagType witten early --- nbt/snbt_decode.go | 92 +++++++++++++++++++++++------------------ nbt/snbt_decode_test.go | 41 ++++++++++++++++-- 2 files changed, 88 insertions(+), 45 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index f450ff8..96e46a9 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -2,6 +2,7 @@ package nbt import ( "bytes" + "errors" "math" "strconv" "strings" @@ -28,48 +29,43 @@ func writeValue(e *Encoder, d *decodeState, tagName string) error { default: panic(phasePanicMsg) case scanBeginLiteral: - return writeLiteral(e, d, tagName) + start := d.readIndex() + d.scanWhile(scanContinue) + literal := d.data[start:d.readIndex()] + tagType, litVal := parseLiteral(literal) + e.writeTag(tagType, tagName) + return writeLiteralPayload(e, litVal) case scanBeginCompound: - return writeCompound(e, d, tagName) + e.writeTag(TagCompound, tagName) + return writeCompoundPayload(e, d) case scanBeginList: return writeListOrArray(e, d, tagName) } } -func writeLiteral(e *Encoder, d *decodeState, tagName string) error { - start := d.readIndex() - d.scanWhile(scanContinue) - literal := d.data[start:d.readIndex()] - switch v := parseLiteral(literal); v.(type) { +func writeLiteralPayload(e *Encoder, v interface{}) error { + switch v.(type) { case string: str := v.(string) - e.writeTag(TagString, tagName) e.writeInt16(int16(len(str))) e.w.Write([]byte(str)) case int8: - e.writeTag(TagByte, tagName) e.w.Write([]byte{byte(v.(int8))}) case int16: - e.writeTag(TagShort, tagName) e.writeInt16(v.(int16)) case int32: - e.writeTag(TagInt, tagName) e.writeInt32(v.(int32)) case int64: - e.writeTag(TagLong, tagName) e.writeInt64(v.(int64)) case float32: - e.writeTag(TagFloat, tagName) e.writeInt32(int32(math.Float32bits(v.(float32)))) case float64: - e.writeTag(TagDouble, tagName) e.writeInt64(int64(math.Float64bits(v.(float64)))) } return nil } -func writeCompound(e *Encoder, d *decodeState, tagName string) error { - e.writeTag(TagCompound, tagName) +func writeCompoundPayload(e *Encoder, d *decodeState) error { for { d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { @@ -114,6 +110,11 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { return nil } + // We don't know the length of the List, + // so we read them into a buffer and count. + var buf bytes.Buffer + var count int + e2 := NewEncoder(&buf) start := d.readIndex() switch d.opcode { @@ -127,27 +128,36 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { if d.opcode != scanListValue { // TAG_List panic(phasePanicMsg) } - // TODO - parseLiteral(literal) - fallthrough - case scanBeginList: // TAG_List - // We don't know the length of the List, - // so we read them into a buffer and count. - var buf bytes.Buffer - e2 := NewEncoder(&buf) - var count int + var tagType byte for { - d.scanWhile(scanSkipSpace) + t, v := parseLiteral(literal) + if tagType == 0 { + tagType = t + } + if t != tagType { + return errors.New("different TagType in List") + } + writeLiteralPayload(e2, v) + count++ + + // read ',' or ']' + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } if d.opcode == scanEndValue { break } - if d.opcode == scanListValue { - // TODO - e2.writeInt32(0) + if d.opcode != scanListValue { + panic(phasePanicMsg) } + start = d.readIndex() + d.scanNext() + d.scanWhile(scanContinue) + literal = d.data[start:d.readIndex()] } - e.writeListHeader(TagList, tagName, count) + e.writeListHeader(tagType, tagName, count) e.w.Write(buf.Bytes()) + case scanBeginList: // TAG_List case scanBeginCompound: // TAG_List } return nil @@ -190,7 +200,7 @@ func (d *decodeState) scanWhile(op int) { // parseLiteral parse an SNBT literal, might be // TAG_String, TAG_Int, TAG_Float, ... etc. // so returned value is one of string, int32, float32 ... -func parseLiteral(literal []byte) interface{} { +func parseLiteral(literal []byte) (byte, interface{}) { switch literal[0] { case '"', '\'': // Quoted String var sb strings.Builder @@ -199,7 +209,7 @@ func parseLiteral(literal []byte) interface{} { c := literal[i] switch c { case literal[0]: - return sb.String() + return TagString, sb.String() case '\\': i++ c = literal[i] @@ -243,17 +253,17 @@ func parseLiteral(literal []byte) interface{} { } switch numberType { case 'B', 'b': - return int8(num) + return TagByte, int8(num) case 'S', 's': - return int16(num) + return TagShort, int16(num) default: - return int32(num) + return TagInt, int32(num) case 'L', 'l': - return num + return TagLong, num case 'F', 'f': - return float32(num) + return TagFloat, float32(num) case 'D', 'd': - return float64(num) + return TagDouble, float64(num) } } else if number { num, err := strconv.ParseFloat(string(literal[:strlen-1]), 64) @@ -262,14 +272,14 @@ func parseLiteral(literal []byte) interface{} { } switch numberType { case 'F', 'f': - return float32(num) + return TagFloat, float32(num) case 'D', 'd': fallthrough default: - return num + return TagDouble, num } } else if unqstr { - return string(literal) + return TagString, string(literal) } } panic(phasePanicMsg) diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 54e9e54..51d31c4 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -8,9 +8,42 @@ import ( func TestEncoder_WriteSNBT(t *testing.T) { var buf bytes.Buffer e := NewEncoder(&buf) - if err := e.WriteSNBT(`[1,2,3]`); err != nil { - t.Fatal(err) - } - t.Log(buf.Bytes()) + testCases := []struct { + snbt string + nbt []byte + }{ + {`10b`, []byte{1, 0, 0, 10}}, + {`12S`, []byte{2, 0, 0, 0, 12}}, + {`0`, []byte{3, 0, 0, 0, 0, 0, 0}}, + {`12L`, []byte{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12}}, + {`""`, []byte{8, 0, 0, 0, 0}}, + {`'""' `, []byte{8, 0, 0, 0, 2, '"', '"'}}, + {`"ab\"c\""`, []byte{8, 0, 0, 0, 5, 'a', 'b', '"', 'c', '"'}}, + {` "1\\23"`, []byte{8, 0, 0, 0, 4, '1', '\\', '2', '3'}}, + + {`{}`, []byte{10, 0, 0, 0}}, + {`{a:1b}`, []byte{10, 0, 0, 1, 0, 1, 'a', 1, 0}}, + {`{ a : 1b }`, []byte{10, 0, 0, 1, 0, 1, 'a', 1, 0}}, + {`{a:1,2:c}`, []byte{10, 0, 0, 3, 0, 1, 'a', 0, 0, 0, 1, 8, 0, 1, '2', 0, 1, 'c', 0}}, + {`{a:{b:{}}}`, []byte{10, 0, 0, 10, 0, 1, 'a', 10, 0, 1, 'b', 0, 0, 0}}, + + {`[]`, []byte{9, 0, 0, 0, 0, 0, 0, 0}}, + {`[1b,2b,3b]`, []byte{9, 0, 0, 1, 0, 0, 0, 3, 1, 2, 3}}, + {`[a,"b",'c']`, []byte{9, 0, 0, 8, 0, 0, 0, 3, 0, 1, 'a', 0, 1, 'b', 0, 1, 'c'}}, + {`[{},{a:1b},{}]`, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, + {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}}, + } + for i := range testCases { + buf.Reset() + if err := e.WriteSNBT(testCases[i].snbt); err != nil { + t.Error(err) + continue + } + want := testCases[i].nbt + got := buf.Bytes() + if !bytes.Equal(want, got) { + t.Errorf("Convert SNBT %q wrong:\nwant: % 02X\ngot: % 02X", testCases[i].snbt, want, got) + } + } } From d0c7a295a5ea9c1342a6d2707a63e070bd2fe217 Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 00:22:26 +0800 Subject: [PATCH 13/23] support TAG_List --- nbt/snbt_decode.go | 26 +++++++++++++++++++++++--- nbt/snbt_decode_test.go | 1 + 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 96e46a9..c6c000d 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -105,8 +105,7 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { // ']', empty TAG_List - e.writeTag(TagList, tagName) - e.writeInt32(0) + e.writeListHeader(TagEnd, tagName, 0) return nil } @@ -150,8 +149,8 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { if d.opcode != scanListValue { panic(phasePanicMsg) } - start = d.readIndex() d.scanNext() + start = d.readIndex() d.scanWhile(scanContinue) literal = d.data[start:d.readIndex()] } @@ -159,6 +158,27 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { e.w.Write(buf.Bytes()) case scanBeginList: // TAG_List case scanBeginCompound: // TAG_List + for { + d.scanWhile(scanSkipSpace) + if d.opcode != scanBeginCompound { + return errors.New("different TagType in List") + } + writeCompoundPayload(e2, d) + count++ + // read ',' or ']' + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + d.scanNext() + if d.opcode == scanEndValue { + break + } + if d.opcode != scanListValue { + panic(phasePanicMsg) + } + } + e.writeListHeader(TagCompound, tagName, count) + e.w.Write(buf.Bytes()) } return nil } diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 51d31c4..83b1889 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -32,6 +32,7 @@ func TestEncoder_WriteSNBT(t *testing.T) { {`[1b,2b,3b]`, []byte{9, 0, 0, 1, 0, 0, 0, 3, 1, 2, 3}}, {`[a,"b",'c']`, []byte{9, 0, 0, 8, 0, 0, 0, 3, 0, 1, 'a', 0, 1, 'b', 0, 1, 'c'}}, {`[{},{a:1b},{}]`, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, + {`[ { } , { a : 1b } , { } ] `, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}}, } for i := range testCases { From 72599f70584e5eab8d63ccf6b991ba1e9cdf7e38 Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 01:03:52 +0800 Subject: [PATCH 14/23] fix TAG_List --- nbt/snbt_decode.go | 11 +++++++++-- nbt/snbt_decode_test.go | 2 +- nbt/snbt_scanner_test.go | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index c6c000d..1b26706 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -159,23 +159,30 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { case scanBeginList: // TAG_List case scanBeginCompound: // TAG_List for { - d.scanWhile(scanSkipSpace) + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } if d.opcode != scanBeginCompound { return errors.New("different TagType in List") } writeCompoundPayload(e2, d) count++ - // read ',' or ']' if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + // read ',' or ']' d.scanNext() + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } if d.opcode == scanEndValue { break } if d.opcode != scanListValue { panic(phasePanicMsg) } + // read '{' + d.scanNext() } e.writeListHeader(TagCompound, tagName, count) e.w.Write(buf.Bytes()) diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 83b1889..8b0aeab 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -38,7 +38,7 @@ func TestEncoder_WriteSNBT(t *testing.T) { for i := range testCases { buf.Reset() if err := e.WriteSNBT(testCases[i].snbt); err != nil { - t.Error(err) + t.Errorf("Convert SNBT %q error: %v", testCases[i].snbt, err) continue } want := testCases[i].nbt diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 5810724..ddbb675 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -9,7 +9,7 @@ func TestSNBT_checkScanCode(t *testing.T) { //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`[I;1,2,3]`) { + for _, c := range []byte(`[{},{a:1b},{}]`) { t.Logf("[%c] - %d", c, s.step(c)) } t.Logf("[%c] - %d", ' ', s.eof()) From 4f148c7515fb958f25fb1c358ec0631a976034cd Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 01:08:11 +0800 Subject: [PATCH 15/23] Add test cases for Typed Array --- nbt/snbt_decode_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 8b0aeab..57d187d 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -34,6 +34,13 @@ func TestEncoder_WriteSNBT(t *testing.T) { {`[{},{a:1b},{}]`, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[ { } , { a : 1b } , { } ] `, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}}, + + {`[B;]`, []byte{7, 0, 0, 0, 0, 0, 0}}, + {`[B;1,2,3]`, []byte{7, 0, 0, 0, 0, 0, 3, 1, 2, 3}}, + {`[I;]`, []byte{11, 0, 0, 0, 0, 0, 0}}, + {`[I;1,2,3]`, []byte{11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3}}, + {`[L;]`, []byte{12, 0, 0, 0, 0, 0, 0}}, + {`[L;1,2,3]`, []byte{12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3}}, } for i := range testCases { buf.Reset() From dd1b8fd9c96c5df276978e772c3579a9fa706fff Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 01:33:08 +0800 Subject: [PATCH 16/23] Support TAG_Byte_Array, TAG_Int_Array and TAG_Long_Array --- nbt/snbt_decode.go | 46 +++++++++++++++++++++++++++++++++++++++- nbt/snbt_scanner.go | 9 +++++++- nbt/snbt_scanner_test.go | 1 + 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 1b26706..0ffa2c8 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -121,7 +121,49 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanContinue) literal := d.data[start:d.readIndex()] if d.opcode == scanListType { // TAG_X_Array - //listType := literal[0] + var elemType byte + switch literal[0] { + case 'B': + e.writeTag(TagByteArray, tagName) + elemType = TagByte + case 'I': + e.writeTag(TagIntArray, tagName) + elemType = TagInt + case 'L': + e.writeTag(TagLongArray, tagName) + elemType = TagLong + } + for { + d.scanNext() + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndValue { // ] + break + } + if d.opcode != scanBeginLiteral { + return errors.New("not literal in Array") + } + start := d.readIndex() + + d.scanWhile(scanContinue) + literal := d.data[start:d.readIndex()] + tagType, litVal := parseLiteral(literal) + if tagType != elemType { + return errors.New("unexpected element type in TAG_Array") + } + switch elemType { + case TagByte: + e2.w.Write([]byte{byte(litVal.(int8))}) + case TagInt: + e2.writeInt32(litVal.(int32)) + case TagLong: + e2.writeInt64(litVal.(int64)) + } + count++ + } + e.writeInt32(int32(count)) + e.w.Write(buf.Bytes()) break } if d.opcode != scanListValue { // TAG_List @@ -157,6 +199,8 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { e.writeListHeader(tagType, tagName, count) e.w.Write(buf.Bytes()) case scanBeginList: // TAG_List + e.writeListHeader(TagList, tagName, count) + e.w.Write(buf.Bytes()) case scanBeginCompound: // TAG_List for { if d.opcode == scanSkipSpace { diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index d97e8d2..44c043f 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -249,12 +249,19 @@ func (s *scanner) stateListOrArray(c byte) int { func (s *scanner) stateListOrArrayT(c byte) int { if c == ';' { - s.step = s.stateBeginValue + s.step = s.stateArrayT return scanListType } return s.stateInUnquotedString(c) } +func (s *scanner) stateArrayT(c byte) int { + if c == ']' { // empty array + return scanEndValue + } + return s.stateBeginValue(c) +} + func (s *scanner) stateNeg(c byte) int { if isNumber(c) { s.step = s.stateNum0 diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index ddbb675..cf5de10 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -67,6 +67,7 @@ func TestSNBT_list(t *testing.T) { `[{}, {}, {"a\"b":520}]`, // List of Compound `[B,C,D]`, `[L, "abc"]`, // List of string (like array) `[B; 01B, 02B, 3B, 10B, 127B]`, // Array + `[I;]`, // Empty array } var s scanner scan := func(str string) bool { From d829d3bccf376911aff1fae48315e7f2ee84f608 Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 10:45:36 +0800 Subject: [PATCH 17/23] support TAG_List --- nbt/encode.go | 10 ++++--- nbt/snbt_decode.go | 66 +++++++++++++++++++++++++++++++---------- nbt/snbt_decode_test.go | 6 ++-- 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/nbt/encode.go b/nbt/encode.go index fbc49b2..6f3836f 100644 --- a/nbt/encode.go +++ b/nbt/encode.go @@ -41,7 +41,7 @@ func (e *Encoder) marshal(val reflect.Value, tagType byte, tagName string) error func (e *Encoder) writeHeader(val reflect.Value, tagType byte, tagName string) (err error) { if tagType == TagList { eleType := getTagType(val.Type().Elem()) - err = e.writeListHeader(eleType, tagName, val.Len()) + err = e.writeListHeader(eleType, tagName, val.Len(), true) } else { err = e.writeTag(tagType, tagName) } @@ -224,9 +224,11 @@ func (e *Encoder) writeTag(tagType byte, tagName string) error { return err } -func (e *Encoder) writeListHeader(elementType byte, tagName string, n int) (err error) { - if err = e.writeTag(TagList, tagName); err != nil { - return +func (e *Encoder) writeListHeader(elementType byte, tagName string, n int, writeTag bool) (err error) { + if writeTag { + if err = e.writeTag(TagList, tagName); err != nil { + return + } } if _, err = e.w.Write([]byte{elementType}); err != nil { return diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 0ffa2c8..21ee862 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -39,7 +39,8 @@ func writeValue(e *Encoder, d *decodeState, tagName string) error { e.writeTag(TagCompound, tagName) return writeCompoundPayload(e, d) case scanBeginList: - return writeListOrArray(e, d, tagName) + _, err := writeListOrArray(e, d, true, tagName) + return err } } @@ -102,11 +103,11 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { return nil } -func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { +func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) (tagType byte, err error) { d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { // ']', empty TAG_List - e.writeListHeader(TagEnd, tagName, 0) - return nil + e.writeListHeader(TagEnd, tagName, 0, writeTag) + return TagList, nil } // We don't know the length of the List, @@ -124,14 +125,19 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { var elemType byte switch literal[0] { case 'B': - e.writeTag(TagByteArray, tagName) + tagType = TagByteArray elemType = TagByte case 'I': - e.writeTag(TagIntArray, tagName) + tagType = TagIntArray elemType = TagInt case 'L': - e.writeTag(TagLongArray, tagName) + tagType = TagLongArray elemType = TagLong + default: + return TagList, errors.New("unknown Array type") + } + if writeTag { + e.writeTag(tagType, tagName) } for { d.scanNext() @@ -142,7 +148,7 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { break } if d.opcode != scanBeginLiteral { - return errors.New("not literal in Array") + return tagType, errors.New("not literal in Array") } start := d.readIndex() @@ -150,7 +156,7 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { literal := d.data[start:d.readIndex()] tagType, litVal := parseLiteral(literal) if tagType != elemType { - return errors.New("unexpected element type in TAG_Array") + return tagType, errors.New("unexpected element type in TAG_Array") } switch elemType { case TagByte: @@ -176,7 +182,7 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { tagType = t } if t != tagType { - return errors.New("different TagType in List") + return TagList, errors.New("different TagType in List") } writeLiteralPayload(e2, v) count++ @@ -196,10 +202,40 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanContinue) literal = d.data[start:d.readIndex()] } - e.writeListHeader(tagType, tagName, count) + e.writeListHeader(tagType, tagName, count, writeTag) e.w.Write(buf.Bytes()) case scanBeginList: // TAG_List - e.writeListHeader(TagList, tagName, count) + var elemType byte + for { + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanBeginList { + return TagList, errors.New("different TagType in List") + } + elemType, err = writeListOrArray(e2, d, false, "") + if err != nil { + return tagType, err + } + count++ + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + // read ',' or ']' + d.scanNext() + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndValue { + break + } + if d.opcode != scanListValue { + panic(phasePanicMsg) + } + // read '[' + d.scanNext() + } + e.writeListHeader(elemType, tagName, count, writeTag) e.w.Write(buf.Bytes()) case scanBeginCompound: // TAG_List for { @@ -207,7 +243,7 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanSkipSpace) } if d.opcode != scanBeginCompound { - return errors.New("different TagType in List") + return TagList, errors.New("different TagType in List") } writeCompoundPayload(e2, d) count++ @@ -228,10 +264,10 @@ func writeListOrArray(e *Encoder, d *decodeState, tagName string) error { // read '{' d.scanNext() } - e.writeListHeader(TagCompound, tagName, count) + e.writeListHeader(TagCompound, tagName, count, writeTag) e.w.Write(buf.Bytes()) } - return nil + return } // readIndex returns the position of the last byte read. diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 57d187d..c9a873c 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -33,14 +33,14 @@ func TestEncoder_WriteSNBT(t *testing.T) { {`[a,"b",'c']`, []byte{9, 0, 0, 8, 0, 0, 0, 3, 0, 1, 'a', 0, 1, 'b', 0, 1, 'c'}}, {`[{},{a:1b},{}]`, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[ { } , { a : 1b } , { } ] `, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, - {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}}, + {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, {`[B;]`, []byte{7, 0, 0, 0, 0, 0, 0}}, - {`[B;1,2,3]`, []byte{7, 0, 0, 0, 0, 0, 3, 1, 2, 3}}, + {`[B;1b,2B,3B]`, []byte{7, 0, 0, 0, 0, 0, 3, 1, 2, 3}}, {`[I;]`, []byte{11, 0, 0, 0, 0, 0, 0}}, {`[I;1,2,3]`, []byte{11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3}}, {`[L;]`, []byte{12, 0, 0, 0, 0, 0, 0}}, - {`[L;1,2,3]`, []byte{12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3}}, + {`[L;1L,2L,3L]`, []byte{12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3}}, } for i := range testCases { buf.Reset() From 82a2efd6a0f4ce217886606f7897510b1104a239 Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 14:19:16 +0800 Subject: [PATCH 18/23] Pass bigTest --- nbt/snbt_decode.go | 48 +++++++++++++++++++++++++++++----------- nbt/snbt_decode_test.go | 28 ++++++++++++++++++----- nbt/snbt_scanner.go | 9 +++++--- nbt/snbt_scanner_test.go | 9 ++++---- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 21ee862..9b8c542 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -67,6 +67,7 @@ func writeLiteralPayload(e *Encoder, v interface{}) error { } func writeCompoundPayload(e *Encoder, d *decodeState) error { + defer d.scanNext() for { d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { @@ -78,7 +79,12 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { // read tag name start := d.readIndex() d.scanWhile(scanContinue) - tagName := string(d.data[start:d.readIndex()]) + var tagName string + if tt, v := parseLiteral(d.data[start:d.readIndex()]); tt == TagString { + tagName = v.(string) + } else { + tagName = string(d.data[start:d.readIndex()]) + } // read value if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) @@ -107,6 +113,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { // ']', empty TAG_List e.writeListHeader(TagEnd, tagName, 0, writeTag) + d.scanNext() return TagList, nil } @@ -121,6 +128,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) case scanBeginLiteral: d.scanWhile(scanContinue) literal := d.data[start:d.readIndex()] + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } if d.opcode == scanListType { // TAG_X_Array var elemType byte switch literal[0] { @@ -139,14 +149,19 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if writeTag { e.writeTag(tagType, tagName) } + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + d.scanWhile(scanSkipSpace) // ; + if d.opcode == scanEndValue { // ] + // empty array + e.writeInt32(0) + break + } for { - d.scanNext() if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } - if d.opcode == scanEndValue { // ] - break - } if d.opcode != scanBeginLiteral { return tagType, errors.New("not literal in Array") } @@ -167,6 +182,17 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) e2.writeInt64(litVal.(int64)) } count++ + + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndValue { // ] + break + } + if d.opcode != scanListValue { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) // , } e.writeInt32(int32(count)) e.w.Write(buf.Bytes()) @@ -197,7 +223,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode != scanListValue { panic(phasePanicMsg) } - d.scanNext() + d.scanWhile(scanSkipSpace) start = d.readIndex() d.scanWhile(scanContinue) literal = d.data[start:d.readIndex()] @@ -221,11 +247,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } - // read ',' or ']' - d.scanNext() - if d.opcode == scanSkipSpace { - d.scanWhile(scanSkipSpace) - } + // ',' or ']' if d.opcode == scanEndValue { break } @@ -251,7 +273,6 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) } // read ',' or ']' - d.scanNext() if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } @@ -267,6 +288,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) e.writeListHeader(TagCompound, tagName, count, writeTag) e.w.Write(buf.Bytes()) } + d.scanNext() return } @@ -334,7 +356,7 @@ func parseLiteral(literal []byte) (byte, interface{}) { if isNumber(c) { continue } else if integer { - if i == strlen-1 && isIntegerType(c) { + if i == strlen-1 && i != 0 && isIntegerType(c) { numberType = c strlen-- } else if i > 0 || i == 0 && c != '-' { diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index c9a873c..523c992 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -25,22 +25,28 @@ func TestEncoder_WriteSNBT(t *testing.T) { {`{}`, []byte{10, 0, 0, 0}}, {`{a:1b}`, []byte{10, 0, 0, 1, 0, 1, 'a', 1, 0}}, {`{ a : 1b }`, []byte{10, 0, 0, 1, 0, 1, 'a', 1, 0}}, - {`{a:1,2:c}`, []byte{10, 0, 0, 3, 0, 1, 'a', 0, 0, 0, 1, 8, 0, 1, '2', 0, 1, 'c', 0}}, - {`{a:{b:{}}}`, []byte{10, 0, 0, 10, 0, 1, 'a', 10, 0, 1, 'b', 0, 0, 0}}, + {`{b:1,2:c}`, []byte{10, 0, 0, 3, 0, 1, 'b', 0, 0, 0, 1, 8, 0, 1, '2', 0, 1, 'c', 0}}, + {`{c:{d:{}}}`, []byte{10, 0, 0, 10, 0, 1, 'c', 10, 0, 1, 'd', 0, 0, 0}}, + {`{h:{},"i":{}}`, []byte{10, 0, 0, 10, 0, 1, 'h', 0, 10, 0, 1, 'i', 0, 0}}, {`[]`, []byte{9, 0, 0, 0, 0, 0, 0, 0}}, {`[1b,2b,3b]`, []byte{9, 0, 0, 1, 0, 0, 0, 3, 1, 2, 3}}, + {`[ 1b , 2b , 3b ]`, []byte{9, 0, 0, 1, 0, 0, 0, 3, 1, 2, 3}}, {`[a,"b",'c']`, []byte{9, 0, 0, 8, 0, 0, 0, 3, 0, 1, 'a', 0, 1, 'b', 0, 1, 'c'}}, {`[{},{a:1b},{}]`, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[ { } , { a : 1b } , { } ] `, []byte{9, 0, 0, 10, 0, 0, 0, 3, 0, 1, 0, 1, 'a', 1, 0, 0}}, {`[[],[]]`, []byte{9, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {`[B;]`, []byte{7, 0, 0, 0, 0, 0, 0}}, - {`[B;1b,2B,3B]`, []byte{7, 0, 0, 0, 0, 0, 3, 1, 2, 3}}, + {`[B; ]`, []byte{7, 0, 0, 0, 0, 0, 0}}, + {`[B; 1b ,2B,3B]`, []byte{7, 0, 0, 0, 0, 0, 3, 1, 2, 3}}, {`[I;]`, []byte{11, 0, 0, 0, 0, 0, 0}}, - {`[I;1,2,3]`, []byte{11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3}}, + {`[I; 1, 2 ,3]`, []byte{11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3}}, {`[L;]`, []byte{12, 0, 0, 0, 0, 0, 0}}, - {`[L;1L,2L,3L]`, []byte{12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3}}, + {`[ L; 1L,2L,3L]`, []byte{12, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3}}, + + {`{d:[]}`, []byte{10, 0, 0, 9, 0, 1, 'd', 0, 0, 0, 0, 0, 0}}, + {`{e:[]}`, []byte{10, 0, 0, 9, 0, 1, 'e', 0, 0, 0, 0, 0, 0}}, + {`{f:[], g:[]}`, []byte{10, 0, 0, 9, 0, 1, 'f', 0, 0, 0, 0, 0, 9, 0, 1, 'g', 0, 0, 0, 0, 0, 0}}, } for i := range testCases { buf.Reset() @@ -55,3 +61,13 @@ func TestEncoder_WriteSNBT(t *testing.T) { } } } + +func TestEncoder_WriteSNBT_bigTest(t *testing.T) { + var buf bytes.Buffer + e := NewEncoder(&buf) + + err := e.WriteSNBT(bigTestSNBT) + if err != nil { + t.Error(err) + } +} diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 44c043f..27aa84d 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -10,10 +10,10 @@ const ( scanBeginLiteral // end implied by next result != scanContinue scanBeginCompound // begin TAG_Compound (after left-brace ) scanBeginList // begin TAG_List (after left-brack) - scanListValue // just finished read list value + scanListValue // just finished read list value (after comma) scanListType // just finished read list type (after "B;" or "L;") scanCompoundTagName // just finished read tag name (before colon) - scanCompoundValue // just finished read value (before comma or right-brace ) + scanCompoundValue // just finished read value (after comma) scanSkipSpace // space byte; can skip; known to be last "continue" result scanEndValue @@ -197,7 +197,7 @@ func (s *scanner) stateInSingleQuotedString(c byte) int { func (s *scanner) stateInSingleQuotedStringEsc(c byte) int { switch c { - case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': + case '\\', '\'': s.step = s.stateInSingleQuotedString return scanContinue } @@ -256,6 +256,9 @@ func (s *scanner) stateListOrArrayT(c byte) int { } func (s *scanner) stateArrayT(c byte) int { + if isSpace(c) { + return scanSkipSpace + } if c == ']' { // empty array return scanEndValue } diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index cf5de10..71eb119 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -9,7 +9,7 @@ func TestSNBT_checkScanCode(t *testing.T) { //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`[{},{a:1b},{}]`) { + for _, c := range []byte(`{ ghi: [B; 2B, 3B], klm: []}`) { t.Logf("[%c] - %d", c, s.step(c)) } t.Logf("[%c] - %d", ' ', s.eof()) @@ -41,13 +41,14 @@ func TestSNBT_number(t *testing.T) { } //go:embed bigTest_test.snbt -var bigTest string +var bigTestSNBT string func TestSNBT_compound(t *testing.T) { goods := []string{ `{}`, `{name:3.14f}`, `{ "name" : 12345 }`, `{ abc: { }}`, `{ "a b\"c": {}, def: 12345}`, - bigTest, + `{ ghi: [], klm: 1}`, + bigTestSNBT, } var s scanner for _, str := range goods { @@ -67,7 +68,7 @@ func TestSNBT_list(t *testing.T) { `[{}, {}, {"a\"b":520}]`, // List of Compound `[B,C,D]`, `[L, "abc"]`, // List of string (like array) `[B; 01B, 02B, 3B, 10B, 127B]`, // Array - `[I;]`, // Empty array + `[I;]`, `[B; ]`, // Empty array } var s scanner scan := func(str string) bool { From 69a8bad0258aea51223db42802e1480118b070f9 Mon Sep 17 00:00:00 2001 From: Tnze Date: Thu, 27 May 2021 14:58:40 +0800 Subject: [PATCH 19/23] fix scanner error of number like "100f" --- nbt/bigTest_test.snbt | 32 ++++++++++++++++---------------- nbt/snbt_scanner.go | 39 ++++----------------------------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/nbt/bigTest_test.snbt b/nbt/bigTest_test.snbt index ec5e3b4..519aeb5 100644 --- a/nbt/bigTest_test.snbt +++ b/nbt/bigTest_test.snbt @@ -1,12 +1,21 @@ { - shortTest: 32767s, longTest: 9223372036854775807L, - byteTest: 127b, - "byteArrayTest (the first 1000 values of (n*n*255+n*7)%100, starting with n=0 (0, 62, 34, 16, 8, ...))": [B; 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B], - "listTest (long)": [11L, 12L, 13L, 14L, 15L], + shortTest: 32767s, + stringTest: "HELLO WORLD THIS IS A TEST STRING ÅÄÖ!", floatTest: 0.49823147f, - doubleTest: 0.4931287132182315d, + byteTest: 127b, intTest: 2147483647, + "nested compound test": { + ham: { + name: "Hampus", + value: 0.75f + }, + egg: { + name: "Eggbert", + value: 0.5f + } + }, + "listTest (long)": [11L, 12L, 13L, 14L, 15L], "listTest (compound)": [ { created-on: 1264099775885L, @@ -17,15 +26,6 @@ name: "Compound tag #1" } ], - "nested compound test": { - egg: { - name: "Eggbert", - value: 0.5f - }, - ham: { - name: "Hampus", - value: 0.75f - } - }, - stringTest: "HELLO WORLD THIS IS A TEST STRING ÅÄÖ!" + "byteArrayTest (the first 1000 values of (n*n*255+n*7)%100, starting with n=0 (0, 62, 34, 16, 8, ...))": [B; 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B, 0B, 62B, 34B, 16B, 8B, 10B, 22B, 44B, 76B, 18B, 70B, 32B, 4B, 86B, 78B, 80B, 92B, 14B, 46B, 88B, 40B, 2B, 74B, 56B, 48B, 50B, 62B, 84B, 16B, 58B, 10B, 72B, 44B, 26B, 18B, 20B, 32B, 54B, 86B, 28B, 80B, 42B, 14B, 96B, 88B, 90B, 2B, 24B, 56B, 98B, 50B, 12B, 84B, 66B, 58B, 60B, 72B, 94B, 26B, 68B, 20B, 82B, 54B, 36B, 28B, 30B, 42B, 64B, 96B, 38B, 90B, 52B, 24B, 6B, 98B, 0B, 12B, 34B, 66B, 8B, 60B, 22B, 94B, 76B, 68B, 70B, 82B, 4B, 36B, 78B, 30B, 92B, 64B, 46B, 38B, 40B, 52B, 74B, 6B, 48B], + doubleTest: 0.4931287132182315d } diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 27aa84d..9aae51f 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -2,7 +2,6 @@ package nbt import ( "errors" - "sync" ) const ( @@ -47,28 +46,6 @@ func (s *scanner) reset() { s.parseState = s.parseState[0:0] } -var scannerPool = sync.Pool{ - New: func() interface{} { - return &scanner{} - }, -} - -func newScanner() *scanner { - scan := scannerPool.Get().(*scanner) - // scan.reset by design doesn't set bytes to zero - //scan.bytes = 0 - scan.reset() - return scan -} - -func freeScanner(scan *scanner) { - // Avoid hanging on to too much memory in extreme cases. - if len(scan.parseState) > 1024 { - scan.parseState = nil - } - scannerPool.Put(scan) -} - // pushParseState pushes a new parse state p onto the parse stack. // an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned. func (s *scanner) pushParseState(c byte, newParseState int, successState int) int { @@ -282,10 +259,6 @@ func (s *scanner) stateNum0(c byte) int { s.step = s.stateNum1 return scanContinue } - if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString - return scanContinue - } return s.stateEndNumValue(c) } @@ -298,10 +271,6 @@ func (s *scanner) stateNum1(c byte) int { s.step = s.stateNumDot return scanContinue } - if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString - return scanContinue - } return s.stateEndNumValue(c) } @@ -326,10 +295,6 @@ func (s *scanner) stateNumDot0(c byte) int { s.step = s.stateNumDot0 return scanContinue } - if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString - return scanContinue - } return s.stateEndNumDotValue(c) } @@ -347,6 +312,10 @@ func (s *scanner) stateEndNumValue(c byte) int { case 'f', 'F', 'd', 'D': return s.stateEndNumDotValue(c) } + if isAllowedInUnquotedString(c) { + s.step = s.stateInUnquotedString + return scanContinue + } return s.stateEndValue(c) } From d678e9b45a4c822a788e3eb264a1cb7d65e35bdd Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 2 Jun 2021 13:35:07 +0800 Subject: [PATCH 20/23] Optimization makes scanner five times faster --- README.md | 66 ++++++++------ nbt/snbt_decode.go | 4 +- nbt/snbt_scanner.go | 183 +++++++++++++++++---------------------- nbt/snbt_scanner_test.go | 24 +++-- 4 files changed, 141 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index a2ab220..90b33c1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Go-MC + ![Version](https://img.shields.io/badge/Minecraft-1.16.5-blue.svg) ![Protocol](https://img.shields.io/badge/Protocol-754-blue.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/Tnze/go-mc.svg)](https://pkg.go.dev/github.com/Tnze/go-mc) @@ -12,18 +13,20 @@ There's some library in Go support you to create your Minecraft client or server - [x] Chat Message (Support Json or old `§`) - [x] NBT (Based on reflection) +- [ ] SNBT -> NBT - [x] Yggdrasil - [x] Realms Server - [x] RCON protocol (Server & Client) - [x] Saves decoding & encoding - [x] Minecraft network protocol -- [x] Robot player framework +- [x] Robot framework > 由于仍在开发中，部分API在未来版本中可能会变动 > `1.13.2` version is at [gomcbot](https://github.com/Tnze/gomcbot). ## Getting start + After you install golang: To get the latest version: `go get github.com/Tnze/go-mc@master` To get old versions (e.g. 1.14.3): `go get github.com/Tnze/go-mc@v1.14.3` @@ -32,12 +35,15 @@ First, you might have a try of the simple examples. It's a good start. ### Run Examples -- Run `go run github.com/Tnze/go-mc/cmd/mcping localhost` to ping and list the localhost mc server. -- Run `go run github.com/Tnze/go-mc/cmd/daze` to join the local server at *localhost:25565* as Steve on the offline mode. +- Run `go run github.com/Tnze/go-mc/cmd/mcping localhost` to ping and list the localhost mc server. +- Run `go run github.com/Tnze/go-mc/cmd/daze` to join the local server at *localhost:25565* as Steve on the offline + mode. ### Basic Usage -One of the most useful functions of this lib is that it implements the network communication protocol of minecraft. It allows you to construct, send, receive, and parse network packets. All of them are encapsulated in `go-mc/net` and `go-mc/net/packet`. +One of the most useful functions of this lib is that it implements the network communication protocol of minecraft. It +allows you to construct, send, receive, and parse network packets. All of them are encapsulated in `go-mc/net` +and `go-mc/net/packet`. 这个库最核心的便是实现了Minecraft底层的网络通信协议，可以用与构造、发送、接收和解读MC数据包。这是靠 `go-mc/net` 和 `go-mc/net/packet`这两个包实现的。 @@ -46,21 +52,23 @@ import "github.com/Tnze/go-mc/net" import pk "github.com/Tnze/go-mc/net/packet" ``` -It's very easy to create a packet. For example, after any client connected the server, it sends a [Handshake Packet](https://wiki.vg/Protocol#Handshake). You can create this package with the following code: +It's very easy to create a packet. For example, after any client connected the server, it sends +a [Handshake Packet](https://wiki.vg/Protocol#Handshake). You can create this package with the following code: 构造一个数据包很简单，例如客户端连接时会发送一个[握手包](https://wiki.vg/Protocol#Handshake)，你就可以用下面这段代码来生成这个包： ```go p := pk.Marshal( - 0x00, // Handshake packet ID - pk.VarInt(ProtocolVersion), // Protocol version - pk.String("localhost"), // Server's address - pk.UnsignedShort(25565), // Server's port - pk.Byte(1), // 1 for status ping, 2 for login +0x00, // Handshake packet ID +pk.VarInt(ProtocolVersion), // Protocol version +pk.String("localhost"), // Server's address +pk.UnsignedShort(25565), // Server's port +pk.Byte(1), // 1 for status ping, 2 for login ) ``` -Then you can send it to server using `conn.WritePacket(p)`. The `conn` is a `net.Conn` which is returned by `net.Dial()`. And don't forget to handle the error.^_^ +Then you can send it to server using `conn.WritePacket(p)`. The `conn` is a `net.Conn` which is returned by `net.Dial()` +. And don't forget to handle the error.^_^ 然后就可以调用`conn.WritePacket(p)`来发送这个p了，其中`conn`是连接对象。发数据包的时候记得不要忘记处理错误噢！ @@ -70,15 +78,15 @@ Receiving packet is quite easy too. To read a packet, call `p.Scan()` like this: ```go var ( - x, y, z pk.Double - yaw, pitch pk.Float - flags pk.Byte - TeleportID pk.VarInt +x, y, z pk.Double +yaw, pitch pk.Float +flags pk.Byte +TeleportID pk.VarInt ) err := p.Scan(&x, &y, &z, &yaw, &pitch, &flags, &TeleportID) if err != nil { - return err +return err } ``` @@ -91,7 +99,8 @@ Sometimes you are handling packet like this: | World Count | VarInt | Size of the following array. | | World Names | Array of Identifier | Identifiers for all worlds on the server. | -That is, the first field is an integer type and the second field is an array (a `[]string` in this case). The integer represents the length of array. +That is, the first field is an integer type and the second field is an array (a `[]string` in this case). The integer +represents the length of array. Traditionally, you can use the following method to read such a field: @@ -100,38 +109,41 @@ r := bytes.Reader(p.Data) // Read WorldCount var WorldCount pk.VarInt if err := WorldCount.ReadFrom(r); err != nil { - return err +return err } // Read WorldNames WorldNames := make([]pk.Identifier, WorldCount) for i := 0; i < int(WorldCount); i++ { - if err := WorldNames[i].ReadFrom(r); err != nil { - return err - } +if err := WorldNames[i].ReadFrom(r); err != nil { +return err +} } ``` But this is tediously long an not compatible with `p.Scan()` method. -In the latest version, two new types is added: `pk.Ary` and `pk.Opt`. Dedicated to handling "Array of ...." and "Optional ...." fields. +In the latest version, two new types is added: `pk.Ary` and `pk.Opt`. Dedicated to handling "Array of ...." and " +Optional ...." fields. ```go var WorldCount pk.VarInt var WorldNames = []pk.Identifier{} if err := p.Scan(&WorldCount, pk.Ary{&WorldCount, &WorldNames}); err != nil { - return err +return err } ``` - - --- -As the `go-mc/net` package implements the minecraft network protocol, there is no update between the versions at this level. So net package actually supports any version. It's just that the ID and content of the package are different between different versions. +As the `go-mc/net` package implements the minecraft network protocol, there is no update between the versions at this +level. So net package actually supports any version. It's just that the ID and content of the package are different +between different versions. 由于`go-mc/net`实现的是MC底层的网络协议，而这个协议在MC更新时其实并不会有改动，MC更新时其实只是包的ID和内容的定义发生了变化，所以net包本身是跨版本的。 -Originally it's all right to write a bot with only `go-mc/net` package, but considering that the process of handshake, login and encryption is not difficult but complicated, I have implemented it in `go-mc/bot` package, which is **not cross-versions**. You may use it directly or as a reference for your own implementation. +Originally it's all right to write a bot with only `go-mc/net` package, but considering that the process of handshake, +login and encryption is not difficult but complicated, I have implemented it in `go-mc/bot` package, which is **not +cross-versions**. You may use it directly or as a reference for your own implementation. 理论上讲，只用`go-mc/net`包实现一个bot是完全可行的，但是为了节省大家从头去理解MC握手、登录、加密等协议的过程，在`go-mc/bot`中我已经把这些都实现了，只不过它不是跨版本的。你可以直接使用，或者作为自己实现的参考。 diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 0ffa2c8..784aa34 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -242,7 +242,7 @@ func (d *decodeState) readIndex() int { // scanNext processes the byte at d.data[d.off]. func (d *decodeState) scanNext() { if d.off < len(d.data) { - d.opcode = d.scan.step(d.data[d.off]) + d.opcode = d.scan.step(&d.scan, d.data[d.off]) d.off++ } else { //d.opcode = d.scan.eof() @@ -255,7 +255,7 @@ func (d *decodeState) scanNext() { func (d *decodeState) scanWhile(op int) { s, data, i := &d.scan, d.data, d.off for i < len(data) { - newOp := s.step(data[i]) + newOp := s.step(s, data[i]) i++ if newOp != op { d.opcode = newOp diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 44c043f..6f918eb 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -2,7 +2,6 @@ package nbt import ( "errors" - "sync" ) const ( @@ -34,7 +33,7 @@ const ( const maxNestingDepth = 10000 type scanner struct { - step func(c byte) int + step func(s *scanner, c byte) int parseState []int err error endTop bool @@ -43,30 +42,10 @@ type scanner struct { // reset prepares the scanner for use. // It must be called before calling s.step. func (s *scanner) reset() { - s.step = s.stateBeginValue + s.step = stateBeginValue s.parseState = s.parseState[0:0] -} - -var scannerPool = sync.Pool{ - New: func() interface{} { - return &scanner{} - }, -} - -func newScanner() *scanner { - scan := scannerPool.Get().(*scanner) - // scan.reset by design doesn't set bytes to zero - //scan.bytes = 0 - scan.reset() - return scan -} - -func freeScanner(scan *scanner) { - // Avoid hanging on to too much memory in extreme cases. - if len(scan.parseState) > 1024 { - scan.parseState = nil - } - scannerPool.Put(scan) + s.err = nil + s.endTop = false } // pushParseState pushes a new parse state p onto the parse stack. @@ -85,10 +64,10 @@ func (s *scanner) popParseState() { n := len(s.parseState) - 1 s.parseState = s.parseState[:n] if n == 0 { - s.step = s.stateEndTop + s.step = stateEndTop s.endTop = true } else { - s.step = s.stateEndValue + s.step = stateEndValue } } @@ -101,7 +80,7 @@ func (s *scanner) eof() int { if s.endTop { return scanEnd } - s.step(' ') + s.step(s, ' ') if s.endTop { return scanEnd } @@ -114,7 +93,7 @@ func (s *scanner) eof() int { // stateEndTop is the state after finishing the top-level value, // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. -func (s *scanner) stateEndTop(c byte) int { +func stateEndTop(s *scanner, c byte) int { if !isSpace(c) { // Complain about non-space byte on next call. s.error(c, "after top-level value") @@ -122,195 +101,195 @@ func (s *scanner) stateEndTop(c byte) int { return scanEnd } -func (s *scanner) stateBeginValue(c byte) int { +func stateBeginValue(s *scanner, c byte) int { if isSpace(c) { - s.step = s.stateBeginValue + s.step = stateBeginValue return scanSkipSpace } switch c { case '{': // beginning of TAG_Compound - s.step = s.stateCompoundOrEmpty + s.step = stateCompoundOrEmpty return s.pushParseState(c, parseCompoundName, scanBeginCompound) case '[': // beginning of TAG_List - s.step = s.stateListOrArray + s.step = stateListOrArray return s.pushParseState(c, parseListValue, scanBeginList) case '"', '\'': // beginning of TAG_String - return s.stateBeginString(c) + return stateBeginString(s, c) case '-': // beginning of negative number - s.step = s.stateNeg + s.step = stateNeg return scanBeginLiteral default: if isNumber(c) { - s.stateNum0(c) + stateNum0(s, c) return scanBeginLiteral } if isAllowedInUnquotedString(c) { - return s.stateBeginString(c) + return stateBeginString(s, c) } } return s.error(c, "looking for beginning of value") } -func (s *scanner) stateCompoundOrEmpty(c byte) int { +func stateCompoundOrEmpty(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } if c == '}' { n := len(s.parseState) s.parseState[n-1] = parseCompoundValue - return s.stateEndValue(c) + return stateEndValue(s, c) } - return s.stateBeginString(c) + return stateBeginString(s, c) } -func (s *scanner) stateBeginString(c byte) int { +func stateBeginString(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } switch c { case '\'': - s.step = s.stateInSingleQuotedString + s.step = stateInSingleQuotedString return scanBeginLiteral case '"': - s.step = s.stateInDoubleQuotedString + s.step = stateInDoubleQuotedString return scanBeginLiteral default: if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanBeginLiteral } } return s.error(c, "looking for beginning of string") } -func (s *scanner) stateInSingleQuotedString(c byte) int { +func stateInSingleQuotedString(s *scanner, c byte) int { if c == '\\' { - s.step = s.stateInSingleQuotedStringEsc + s.step = stateInSingleQuotedStringEsc return scanContinue } if c == '\'' { - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } return scanContinue } -func (s *scanner) stateInSingleQuotedStringEsc(c byte) int { +func stateInSingleQuotedStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': - s.step = s.stateInSingleQuotedString + s.step = stateInSingleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInDoubleQuotedString(c byte) int { +func stateInDoubleQuotedString(s *scanner, c byte) int { if c == '\\' { - s.step = s.stateInDqStringEsc + s.step = stateInDqStringEsc return scanContinue } if c == '"' { - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } return scanContinue } -func (s *scanner) stateInDqStringEsc(c byte) int { +func stateInDqStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': - s.step = s.stateInDoubleQuotedString + s.step = stateInDoubleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInUnquotedString(c byte) int { +func stateInUnquotedString(s *scanner, c byte) int { if isAllowedInUnquotedString(c) { return scanContinue } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateListOrArray(c byte) int { +func stateListOrArray(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } switch c { case 'B', 'I', 'L': - s.step = s.stateListOrArrayT + s.step = stateListOrArrayT return scanBeginLiteral case ']': - return s.stateEndValue(c) + return stateEndValue(s, c) default: - return s.stateBeginValue(c) + return stateBeginValue(s, c) } } -func (s *scanner) stateListOrArrayT(c byte) int { +func stateListOrArrayT(s *scanner, c byte) int { if c == ';' { - s.step = s.stateArrayT + s.step = stateArrayT return scanListType } - return s.stateInUnquotedString(c) + return stateInUnquotedString(s, c) } -func (s *scanner) stateArrayT(c byte) int { +func stateArrayT(s *scanner, c byte) int { if c == ']' { // empty array return scanEndValue } - return s.stateBeginValue(c) + return stateBeginValue(s, c) } -func (s *scanner) stateNeg(c byte) int { +func stateNeg(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum0 + s.step = stateNum0 return scanBeginLiteral } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanBeginLiteral } return s.error(c, "not a number after '-'") } -func (s *scanner) stateNum0(c byte) int { +func stateNum0(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum1 + s.step = stateNum1 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumValue(c) + return stateEndNumValue(s, c) } -func (s *scanner) stateNum1(c byte) int { +func stateNum1(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum1 + s.step = stateNum1 return scanContinue } if c == '.' { - s.step = s.stateNumDot + s.step = stateNumDot return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumValue(c) + return stateEndNumValue(s, c) } // stateDot is the state after reading the integer and decimal point in a number, // such as after reading `1.`. -func (s *scanner) stateNumDot(c byte) int { +func stateNumDot(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNumDot0 + s.step = stateNumDot0 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } return s.error(c, "after decimal point in numeric literal") @@ -318,54 +297,54 @@ func (s *scanner) stateNumDot(c byte) int { // stateNumDot0 is the state after reading the integer, decimal point, and subsequent // digits of a number, such as after reading `3.14`. -func (s *scanner) stateNumDot0(c byte) int { +func stateNumDot0(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNumDot0 + s.step = stateNumDot0 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumDotValue(c) + return stateEndNumDotValue(s, c) } -func (s *scanner) stateEndNumValue(c byte) int { +func stateEndNumValue(s *scanner, c byte) int { switch c { case 'b', 'B': // TAG_Byte - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 's', 'S': // TAG_Short - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'l', 'L': // TAG_Long - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'f', 'F', 'd', 'D': - return s.stateEndNumDotValue(c) + return stateEndNumDotValue(s, c) } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateEndNumDotValue(c byte) int { +func stateEndNumDotValue(s *scanner, c byte) int { switch c { case 'f', 'F': // TAG_Float - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'd', 'D': // TAG_Double - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateEndValue(c byte) int { +func stateEndValue(s *scanner, c byte) int { n := len(s.parseState) if n == 0 { // Completed top-level before the current byte. - s.step = s.stateEndTop + s.step = stateEndTop s.endTop = true - return s.stateEndTop(c) + return stateEndTop(s, c) } if isSpace(c) { return scanSkipSpace @@ -376,7 +355,7 @@ func (s *scanner) stateEndValue(c byte) int { case parseCompoundName: if c == ':' { s.parseState[n-1] = parseCompoundValue - s.step = s.stateBeginValue + s.step = stateBeginValue return scanCompoundTagName } return s.error(c, "after compound tag name") @@ -384,7 +363,7 @@ func (s *scanner) stateEndValue(c byte) int { switch c { case ',': s.parseState[n-1] = parseCompoundName - s.step = s.stateBeginString + s.step = stateBeginString return scanCompoundValue case '}': s.popParseState() @@ -394,7 +373,7 @@ func (s *scanner) stateEndValue(c byte) int { case parseListValue: switch c { case ',': - s.step = s.stateBeginValue + s.step = stateBeginValue return scanListValue case ']': s.popParseState() @@ -406,14 +385,14 @@ func (s *scanner) stateEndValue(c byte) int { } func (s *scanner) error(c byte, context string) int { - s.step = s.stateError + s.step = stateError s.err = errors.New(context) return scanError } // stateError is the state after reaching a syntax error, // such as after reading `[1}` or `5.1.2`. -func (s *scanner) stateError(c byte) int { +func stateError(s *scanner, c byte) int { return scanError } diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index cf5de10..b22d520 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -9,8 +9,8 @@ func TestSNBT_checkScanCode(t *testing.T) { //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`[{},{a:1b},{}]`) { - t.Logf("[%c] - %d", c, s.step(c)) + for _, c := range []byte(`[I;123,345]`) { + t.Logf("[%c] - %d", c, s.step(&s, c)) } t.Logf("[%c] - %d", ' ', s.eof()) } @@ -26,7 +26,7 @@ func TestSNBT_number(t *testing.T) { scan := func(str string) bool { s.reset() for _, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { return false } @@ -53,7 +53,7 @@ func TestSNBT_compound(t *testing.T) { for _, str := range goods { s.reset() for i, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { t.Errorf("scan valid data %q error: %v at [%d]", str[:i], s.err, i) break @@ -73,7 +73,7 @@ func TestSNBT_list(t *testing.T) { scan := func(str string) bool { s.reset() for _, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { return false } @@ -86,3 +86,17 @@ func TestSNBT_list(t *testing.T) { } } } + +func BenchmarkSNBT_bigTest(b *testing.B) { + var s scanner + for i := 0; i < b.N; i++ { + s.reset() + for _, c := range []byte(bigTest) { + res := s.step(&s, c) + if res == scanError { + b.Errorf("scan valid data %q error: %v at [%d]", bigTest[:i], s.err, i) + break + } + } + } +} From d73015375049f010f3e198dae60eb29f77447288 Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 9 Jun 2021 23:43:12 +0800 Subject: [PATCH 21/23] Add Benchmark --- nbt/snbt_decode_test.go | 12 ++++++++++++ nbt/snbt_scanner_test.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 523c992..9f74560 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -71,3 +71,15 @@ func TestEncoder_WriteSNBT_bigTest(t *testing.T) { t.Error(err) } } + +func BenchmarkEncoder_WriteSNBT_bigTest(b *testing.B) { + var buf bytes.Buffer + e := NewEncoder(&buf) + for i := 0; i < b.N; i++ { + err := e.WriteSNBT(bigTestSNBT) + if err != nil { + b.Fatal(err) + } + buf.Reset() + } +} diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index ed4bc1b..2036564 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -9,7 +9,7 @@ func TestSNBT_checkScanCode(t *testing.T) { //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`[I;123,345]`) { + for _, c := range []byte(`[I;123,345], `) { t.Logf("[%c] - %d", c, s.step(&s, c)) } t.Logf("[%c] - %d", ' ', s.eof()) From 723303ce8d408290cacabe22dbe897de0ba5cdf2 Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 22 Jun 2021 10:01:41 +0800 Subject: [PATCH 22/23] Better syntax error handling --- nbt/snbt_decode.go | 75 +++++++++++++++++++++++++++++++++------- nbt/snbt_decode_test.go | 26 ++++++++++++++ nbt/snbt_scanner.go | 33 ++++++++++++------ nbt/snbt_scanner_test.go | 8 ++--- 4 files changed, 115 insertions(+), 27 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 266f4e0..912df6b 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -2,7 +2,6 @@ package nbt import ( "bytes" - "errors" "math" "strconv" "strings" @@ -10,7 +9,7 @@ import ( type decodeState struct { data []byte - off int // next read offset in data + off int // next read Offset in data opcode int // last read result scan scanner } @@ -26,18 +25,25 @@ func (e *Encoder) WriteSNBT(snbt string) error { func writeValue(e *Encoder, d *decodeState, tagName string) error { d.scanWhile(scanSkipSpace) switch d.opcode { + case scanError: + return d.error(d.scan.errContext) default: panic(phasePanicMsg) + case scanBeginLiteral: start := d.readIndex() - d.scanWhile(scanContinue) + if d.scanWhile(scanContinue); d.opcode == scanError { + return d.error(d.scan.errContext) + } literal := d.data[start:d.readIndex()] tagType, litVal := parseLiteral(literal) e.writeTag(tagType, tagName) return writeLiteralPayload(e, litVal) + case scanBeginCompound: e.writeTag(TagCompound, tagName) return writeCompoundPayload(e, d) + case scanBeginList: _, err := writeListOrArray(e, d, true, tagName) return err @@ -73,12 +79,17 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { if d.opcode == scanEndValue { break } + if d.opcode == scanError { + return d.error(d.scan.errContext) + } if d.opcode != scanBeginLiteral { panic(phasePanicMsg) } // read tag name start := d.readIndex() - d.scanWhile(scanContinue) + if d.scanWhile(scanContinue); d.opcode == scanError { + return d.error(d.scan.errContext) + } var tagName string if tt, v := parseLiteral(d.data[start:d.readIndex()]); tt == TagString { tagName = v.(string) @@ -89,6 +100,9 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return d.error(d.scan.errContext) + } if d.opcode != scanCompoundTagName { panic(phasePanicMsg) } @@ -98,6 +112,9 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return d.error(d.scan.errContext) + } if d.opcode == scanEndValue { break } @@ -126,11 +143,16 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) switch d.opcode { case scanBeginLiteral: - d.scanWhile(scanContinue) + if d.scanWhile(scanContinue); d.opcode == scanError { + return TagList, d.error(d.scan.errContext) + } literal := d.data[start:d.readIndex()] if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } if d.opcode == scanListType { // TAG_X_Array var elemType byte switch literal[0] { @@ -144,7 +166,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) tagType = TagLongArray elemType = TagLong default: - return TagList, errors.New("unknown Array type") + return TagList, d.error("unknown Array type") } if writeTag { e.writeTag(tagType, tagName) @@ -163,15 +185,17 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) } if d.opcode != scanBeginLiteral { - return tagType, errors.New("not literal in Array") + return tagType, d.error("not literal in Array") } start := d.readIndex() - d.scanWhile(scanContinue) + if d.scanWhile(scanContinue); d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } literal := d.data[start:d.readIndex()] tagType, litVal := parseLiteral(literal) if tagType != elemType { - return tagType, errors.New("unexpected element type in TAG_Array") + return tagType, d.error("unexpected element type in TAG_Array") } switch elemType { case TagByte: @@ -186,6 +210,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } if d.opcode == scanEndValue { // ] break } @@ -208,7 +235,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) tagType = t } if t != tagType { - return TagList, errors.New("different TagType in List") + return TagList, d.error("different TagType in List") } writeLiteralPayload(e2, v) count++ @@ -217,6 +244,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } if d.opcode == scanEndValue { break } @@ -225,7 +255,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) } d.scanWhile(scanSkipSpace) start = d.readIndex() - d.scanWhile(scanContinue) + if d.scanWhile(scanContinue); d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } literal = d.data[start:d.readIndex()] } e.writeListHeader(tagType, tagName, count, writeTag) @@ -237,7 +269,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) } if d.opcode != scanBeginList { - return TagList, errors.New("different TagType in List") + return TagList, d.error("different TagType in List") } elemType, err = writeListOrArray(e2, d, false, "") if err != nil { @@ -247,6 +279,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } // ',' or ']' if d.opcode == scanEndValue { break @@ -265,7 +300,7 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) } if d.opcode != scanBeginCompound { - return TagList, errors.New("different TagType in List") + return TagList, d.error("different TagType in List") } writeCompoundPayload(e2, d) count++ @@ -276,6 +311,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) } + if d.opcode == scanError { + return tagType, d.error(d.scan.errContext) + } if d.opcode == scanEndValue { break } @@ -414,6 +452,10 @@ func parseLiteral(literal []byte) (byte, interface{}) { panic(phasePanicMsg) } +func (d *decodeState) error(msg string) *SyntaxError { + return &SyntaxError{Message: msg, Offset: d.off} +} + func isIntegerType(c byte) bool { return isFloatType(c) || c == 'B' || c == 'b' || @@ -424,3 +466,10 @@ func isIntegerType(c byte) bool { func isFloatType(c byte) bool { return c == 'F' || c == 'f' || c == 'D' || c == 'd' } + +type SyntaxError struct { + Message string + Offset int +} + +func (e *SyntaxError) Error() string { return e.Message } diff --git a/nbt/snbt_decode_test.go b/nbt/snbt_decode_test.go index 9f74560..a683d18 100644 --- a/nbt/snbt_decode_test.go +++ b/nbt/snbt_decode_test.go @@ -2,6 +2,7 @@ package nbt import ( "bytes" + "strings" "testing" ) @@ -83,3 +84,28 @@ func BenchmarkEncoder_WriteSNBT_bigTest(b *testing.B) { buf.Reset() } } + +func Test_WriteSNBT_nestingList(t *testing.T) { + var buf bytes.Buffer + e := NewEncoder(&buf) + + // Our maximum supported nesting depth is 10000. + // The nesting depth of 10001 is 10000 + err := e.WriteSNBT(strings.Repeat("[", 10001) + strings.Repeat("]", 10001)) + if err != nil { + t.Error(err) + } + + // Following code should return error instant of panic. + buf.Reset() + err = e.WriteSNBT(strings.Repeat("[", 10002) + strings.Repeat("]", 10002)) + if err == nil { + t.Error("Exceeded the maximum depth of support, but no error was reported") + } + // Panic test + buf.Reset() + err = e.WriteSNBT(strings.Repeat("[", 20000) + strings.Repeat("]", 20000)) + if err == nil { + t.Error("Exceeded the maximum depth of support, but no error was reported") + } +} diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index d057794..17c5723 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -1,8 +1,6 @@ package nbt -import ( - "errors" -) +import "strconv" const ( scanContinue = iota // uninteresting byte @@ -35,7 +33,7 @@ const maxNestingDepth = 10000 type scanner struct { step func(s *scanner, c byte) int parseState []int - err error + errContext string endTop bool } @@ -44,7 +42,7 @@ type scanner struct { func (s *scanner) reset() { s.step = stateBeginValue s.parseState = s.parseState[0:0] - s.err = nil + s.errContext = "" s.endTop = false } @@ -74,7 +72,7 @@ func (s *scanner) popParseState() { // eof tells the scanner that the end of input has been reached. // It returns a scan status just as s.step does. func (s *scanner) eof() int { - if s.err != nil { + if s.errContext != "" { return scanError } if s.endTop { @@ -84,8 +82,8 @@ func (s *scanner) eof() int { if s.endTop { return scanEnd } - if s.err == nil { - s.err = errors.New("unexpected end of JSON input") + if s.errContext == "" { + s.errContext = "unexpected end of JSON input" } return scanError } @@ -381,13 +379,13 @@ func stateEndValue(s *scanner, c byte) int { func (s *scanner) error(c byte, context string) int { s.step = stateError - s.err = errors.New(context) + s.errContext = "invalid character " + quoteChar(c) + " " + context return scanError } // stateError is the state after reaching a syntax error, // such as after reading `[1}` or `5.1.2`. -func stateError(s *scanner, c byte) int { +func stateError(*scanner, byte) int { return scanError } @@ -406,3 +404,18 @@ func isAllowedInUnquotedString(c byte) bool { c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' } + +// quoteChar formats c as a quoted character literal +func quoteChar(c byte) string { + // special cases - different from quoted strings + if c == '\'' { + return `'\''` + } + if c == '"' { + return `'"'` + } + + // use quoted string with different quotation marks + s := strconv.Quote(string(c)) + return "'" + s[1:len(s)-1] + "'" +} diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index 2036564..9bf7b37 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -35,7 +35,7 @@ func TestSNBT_number(t *testing.T) { } for _, str := range goods { if scan(str) == false { - t.Errorf("scan valid data %q error: %v", str, s.err) + t.Errorf("scan valid data %q error: %v", str, s.errContext) } } } @@ -56,7 +56,7 @@ func TestSNBT_compound(t *testing.T) { for i, c := range []byte(str) { res := s.step(&s, c) if res == scanError { - t.Errorf("scan valid data %q error: %v at [%d]", str[:i], s.err, i) + t.Errorf("scan valid data %q error: %v at [%d]", str[:i], s.errContext, i) break } } @@ -83,7 +83,7 @@ func TestSNBT_list(t *testing.T) { } for _, str := range goods { if scan(str) == false { - t.Errorf("scan valid data %q error: %v", str, s.err) + t.Errorf("scan valid data %q error: %v", str, s.errContext) } } } @@ -95,7 +95,7 @@ func BenchmarkSNBT_bigTest(b *testing.B) { for _, c := range []byte(bigTestSNBT) { res := s.step(&s, c) if res == scanError { - b.Errorf("scan valid data %q error: %v at [%d]", bigTestSNBT[:i], s.err, i) + b.Errorf("scan valid data %q error: %v at [%d]", bigTestSNBT[:i], s.errContext, i) break } } From 2ff54efb7d2c885942083c0f8273aa05f2a4a55b Mon Sep 17 00:00:00 2001 From: Tnze Date: Tue, 22 Jun 2021 10:33:07 +0800 Subject: [PATCH 23/23] Better io error handling --- nbt/snbt_decode.go | 116 +++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 35 deletions(-) diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 912df6b..282b8e6 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -37,11 +37,15 @@ func writeValue(e *Encoder, d *decodeState, tagName string) error { } literal := d.data[start:d.readIndex()] tagType, litVal := parseLiteral(literal) - e.writeTag(tagType, tagName) + if err := e.writeTag(tagType, tagName); err != nil { + return err + } return writeLiteralPayload(e, litVal) case scanBeginCompound: - e.writeTag(TagCompound, tagName) + if err := e.writeTag(TagCompound, tagName); err != nil { + return err + } return writeCompoundPayload(e, d) case scanBeginList: @@ -50,26 +54,29 @@ func writeValue(e *Encoder, d *decodeState, tagName string) error { } } -func writeLiteralPayload(e *Encoder, v interface{}) error { +func writeLiteralPayload(e *Encoder, v interface{}) (err error) { switch v.(type) { case string: str := v.(string) - e.writeInt16(int16(len(str))) - e.w.Write([]byte(str)) + err = e.writeInt16(int16(len(str))) + if err != nil { + return + } + _, err = e.w.Write([]byte(str)) case int8: - e.w.Write([]byte{byte(v.(int8))}) + _, err = e.w.Write([]byte{byte(v.(int8))}) case int16: - e.writeInt16(v.(int16)) + err = e.writeInt16(v.(int16)) case int32: - e.writeInt32(v.(int32)) + err = e.writeInt32(v.(int32)) case int64: - e.writeInt64(v.(int64)) + err = e.writeInt64(v.(int64)) case float32: - e.writeInt32(int32(math.Float32bits(v.(float32)))) + err = e.writeInt32(int32(math.Float32bits(v.(float32)))) case float64: - e.writeInt64(int64(math.Float64bits(v.(float64)))) + err = e.writeInt64(int64(math.Float64bits(v.(float64)))) } - return nil + return } func writeCompoundPayload(e *Encoder, d *decodeState) error { @@ -106,7 +113,10 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { if d.opcode != scanCompoundTagName { panic(phasePanicMsg) } - writeValue(e, d, tagName) + + if err := writeValue(e, d, tagName); err != nil { + return err + } // Next token must be , or }. if d.opcode == scanSkipSpace { @@ -122,16 +132,16 @@ func writeCompoundPayload(e *Encoder, d *decodeState) error { panic(phasePanicMsg) } } - e.w.Write([]byte{TagEnd}) - return nil + _, err := e.w.Write([]byte{TagEnd}) + return err } func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) (tagType byte, err error) { d.scanWhile(scanSkipSpace) if d.opcode == scanEndValue { // ']', empty TAG_List - e.writeListHeader(TagEnd, tagName, 0, writeTag) + err = e.writeListHeader(TagEnd, tagName, 0, writeTag) d.scanNext() - return TagList, nil + return TagList, err } // We don't know the length of the List, @@ -169,7 +179,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) return TagList, d.error("unknown Array type") } if writeTag { - e.writeTag(tagType, tagName) + if err = e.writeTag(tagType, tagName); err != nil { + return + } } if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) @@ -177,7 +189,9 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) d.scanWhile(scanSkipSpace) // ; if d.opcode == scanEndValue { // ] // empty array - e.writeInt32(0) + if err = e.writeInt32(0); err != nil { + return + } break } for { @@ -193,17 +207,21 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) return tagType, d.error(d.scan.errContext) } literal := d.data[start:d.readIndex()] - tagType, litVal := parseLiteral(literal) - if tagType != elemType { - return tagType, d.error("unexpected element type in TAG_Array") + subType, litVal := parseLiteral(literal) + if subType != elemType { + err = d.error("unexpected element type in TAG_Array") + return } switch elemType { case TagByte: - e2.w.Write([]byte{byte(litVal.(int8))}) + _, err = e2.w.Write([]byte{byte(litVal.(int8))}) case TagInt: - e2.writeInt32(litVal.(int32)) + err = e2.writeInt32(litVal.(int32)) case TagLong: - e2.writeInt64(litVal.(int64)) + err = e2.writeInt64(litVal.(int64)) + } + if err != nil { + return } count++ @@ -221,8 +239,14 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) } d.scanWhile(scanSkipSpace) // , } - e.writeInt32(int32(count)) - e.w.Write(buf.Bytes()) + + if err = e.writeInt32(int32(count)); err != nil { + return tagType, err + } + _, err = e.w.Write(buf.Bytes()) + if err != nil { + return tagType, err + } break } if d.opcode != scanListValue { // TAG_List @@ -237,7 +261,10 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if t != tagType { return TagList, d.error("different TagType in List") } - writeLiteralPayload(e2, v) + err = writeLiteralPayload(e2, v) + if err != nil { + return tagType, err + } count++ // read ',' or ']' @@ -260,8 +287,13 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) } literal = d.data[start:d.readIndex()] } - e.writeListHeader(tagType, tagName, count, writeTag) - e.w.Write(buf.Bytes()) + + if err := e.writeListHeader(tagType, tagName, count, writeTag); err != nil { + return tagType, err + } + if _, err := e.w.Write(buf.Bytes()); err != nil { + return tagType, err + } case scanBeginList: // TAG_List var elemType byte for { @@ -292,8 +324,13 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) // read '[' d.scanNext() } - e.writeListHeader(elemType, tagName, count, writeTag) - e.w.Write(buf.Bytes()) + + if err = e.writeListHeader(elemType, tagName, count, writeTag); err != nil { + return + } + if _, err = e.w.Write(buf.Bytes()); err != nil { + return + } case scanBeginCompound: // TAG_List for { if d.opcode == scanSkipSpace { @@ -302,7 +339,10 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) if d.opcode != scanBeginCompound { return TagList, d.error("different TagType in List") } - writeCompoundPayload(e2, d) + + if err = writeCompoundPayload(e2, d); err != nil { + return + } count++ if d.opcode == scanSkipSpace { d.scanWhile(scanSkipSpace) @@ -323,8 +363,14 @@ func writeListOrArray(e *Encoder, d *decodeState, writeTag bool, tagName string) // read '{' d.scanNext() } - e.writeListHeader(TagCompound, tagName, count, writeTag) - e.w.Write(buf.Bytes()) + + if err = e.writeListHeader(TagCompound, tagName, count, writeTag); err != nil { + return + } + + if _, err = e.w.Write(buf.Bytes()); err != nil { + return + } } d.scanNext() return