From d678e9b45a4c822a788e3eb264a1cb7d65e35bdd Mon Sep 17 00:00:00 2001 From: Tnze Date: Wed, 2 Jun 2021 13:35:07 +0800 Subject: [PATCH] Optimization makes scanner five times faster --- README.md | 66 ++++++++------ nbt/snbt_decode.go | 4 +- nbt/snbt_scanner.go | 183 +++++++++++++++++---------------------- nbt/snbt_scanner_test.go | 24 +++-- 4 files changed, 141 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index a2ab220..90b33c1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Go-MC + ![Version](https://img.shields.io/badge/Minecraft-1.16.5-blue.svg) ![Protocol](https://img.shields.io/badge/Protocol-754-blue.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/Tnze/go-mc.svg)](https://pkg.go.dev/github.com/Tnze/go-mc) @@ -12,18 +13,20 @@ There's some library in Go support you to create your Minecraft client or server - [x] Chat Message (Support Json or old `§`) - [x] NBT (Based on reflection) +- [ ] SNBT -> NBT - [x] Yggdrasil - [x] Realms Server - [x] RCON protocol (Server & Client) - [x] Saves decoding & encoding - [x] Minecraft network protocol -- [x] Robot player framework +- [x] Robot framework > 由于仍在开发中，部分API在未来版本中可能会变动 > `1.13.2` version is at [gomcbot](https://github.com/Tnze/gomcbot). ## Getting start + After you install golang: To get the latest version: `go get github.com/Tnze/go-mc@master` To get old versions (e.g. 1.14.3): `go get github.com/Tnze/go-mc@v1.14.3` @@ -32,12 +35,15 @@ First, you might have a try of the simple examples. It's a good start. ### Run Examples -- Run `go run github.com/Tnze/go-mc/cmd/mcping localhost` to ping and list the localhost mc server. -- Run `go run github.com/Tnze/go-mc/cmd/daze` to join the local server at *localhost:25565* as Steve on the offline mode. +- Run `go run github.com/Tnze/go-mc/cmd/mcping localhost` to ping and list the localhost mc server. +- Run `go run github.com/Tnze/go-mc/cmd/daze` to join the local server at *localhost:25565* as Steve on the offline + mode. ### Basic Usage -One of the most useful functions of this lib is that it implements the network communication protocol of minecraft. It allows you to construct, send, receive, and parse network packets. All of them are encapsulated in `go-mc/net` and `go-mc/net/packet`. +One of the most useful functions of this lib is that it implements the network communication protocol of minecraft. It +allows you to construct, send, receive, and parse network packets. All of them are encapsulated in `go-mc/net` +and `go-mc/net/packet`. 这个库最核心的便是实现了Minecraft底层的网络通信协议，可以用与构造、发送、接收和解读MC数据包。这是靠 `go-mc/net` 和 `go-mc/net/packet`这两个包实现的。 @@ -46,21 +52,23 @@ import "github.com/Tnze/go-mc/net" import pk "github.com/Tnze/go-mc/net/packet" ``` -It's very easy to create a packet. For example, after any client connected the server, it sends a [Handshake Packet](https://wiki.vg/Protocol#Handshake). You can create this package with the following code: +It's very easy to create a packet. For example, after any client connected the server, it sends +a [Handshake Packet](https://wiki.vg/Protocol#Handshake). You can create this package with the following code: 构造一个数据包很简单，例如客户端连接时会发送一个[握手包](https://wiki.vg/Protocol#Handshake)，你就可以用下面这段代码来生成这个包： ```go p := pk.Marshal( - 0x00, // Handshake packet ID - pk.VarInt(ProtocolVersion), // Protocol version - pk.String("localhost"), // Server's address - pk.UnsignedShort(25565), // Server's port - pk.Byte(1), // 1 for status ping, 2 for login +0x00, // Handshake packet ID +pk.VarInt(ProtocolVersion), // Protocol version +pk.String("localhost"), // Server's address +pk.UnsignedShort(25565), // Server's port +pk.Byte(1), // 1 for status ping, 2 for login ) ``` -Then you can send it to server using `conn.WritePacket(p)`. The `conn` is a `net.Conn` which is returned by `net.Dial()`. And don't forget to handle the error.^_^ +Then you can send it to server using `conn.WritePacket(p)`. The `conn` is a `net.Conn` which is returned by `net.Dial()` +. And don't forget to handle the error.^_^ 然后就可以调用`conn.WritePacket(p)`来发送这个p了，其中`conn`是连接对象。发数据包的时候记得不要忘记处理错误噢！ @@ -70,15 +78,15 @@ Receiving packet is quite easy too. To read a packet, call `p.Scan()` like this: ```go var ( - x, y, z pk.Double - yaw, pitch pk.Float - flags pk.Byte - TeleportID pk.VarInt +x, y, z pk.Double +yaw, pitch pk.Float +flags pk.Byte +TeleportID pk.VarInt ) err := p.Scan(&x, &y, &z, &yaw, &pitch, &flags, &TeleportID) if err != nil { - return err +return err } ``` @@ -91,7 +99,8 @@ Sometimes you are handling packet like this: | World Count | VarInt | Size of the following array. | | World Names | Array of Identifier | Identifiers for all worlds on the server. | -That is, the first field is an integer type and the second field is an array (a `[]string` in this case). The integer represents the length of array. +That is, the first field is an integer type and the second field is an array (a `[]string` in this case). The integer +represents the length of array. Traditionally, you can use the following method to read such a field: @@ -100,38 +109,41 @@ r := bytes.Reader(p.Data) // Read WorldCount var WorldCount pk.VarInt if err := WorldCount.ReadFrom(r); err != nil { - return err +return err } // Read WorldNames WorldNames := make([]pk.Identifier, WorldCount) for i := 0; i < int(WorldCount); i++ { - if err := WorldNames[i].ReadFrom(r); err != nil { - return err - } +if err := WorldNames[i].ReadFrom(r); err != nil { +return err +} } ``` But this is tediously long an not compatible with `p.Scan()` method. -In the latest version, two new types is added: `pk.Ary` and `pk.Opt`. Dedicated to handling "Array of ...." and "Optional ...." fields. +In the latest version, two new types is added: `pk.Ary` and `pk.Opt`. Dedicated to handling "Array of ...." and " +Optional ...." fields. ```go var WorldCount pk.VarInt var WorldNames = []pk.Identifier{} if err := p.Scan(&WorldCount, pk.Ary{&WorldCount, &WorldNames}); err != nil { - return err +return err } ``` - - --- -As the `go-mc/net` package implements the minecraft network protocol, there is no update between the versions at this level. So net package actually supports any version. It's just that the ID and content of the package are different between different versions. +As the `go-mc/net` package implements the minecraft network protocol, there is no update between the versions at this +level. So net package actually supports any version. It's just that the ID and content of the package are different +between different versions. 由于`go-mc/net`实现的是MC底层的网络协议，而这个协议在MC更新时其实并不会有改动，MC更新时其实只是包的ID和内容的定义发生了变化，所以net包本身是跨版本的。 -Originally it's all right to write a bot with only `go-mc/net` package, but considering that the process of handshake, login and encryption is not difficult but complicated, I have implemented it in `go-mc/bot` package, which is **not cross-versions**. You may use it directly or as a reference for your own implementation. +Originally it's all right to write a bot with only `go-mc/net` package, but considering that the process of handshake, +login and encryption is not difficult but complicated, I have implemented it in `go-mc/bot` package, which is **not +cross-versions**. You may use it directly or as a reference for your own implementation. 理论上讲，只用`go-mc/net`包实现一个bot是完全可行的，但是为了节省大家从头去理解MC握手、登录、加密等协议的过程，在`go-mc/bot`中我已经把这些都实现了，只不过它不是跨版本的。你可以直接使用，或者作为自己实现的参考。 diff --git a/nbt/snbt_decode.go b/nbt/snbt_decode.go index 0ffa2c8..784aa34 100644 --- a/nbt/snbt_decode.go +++ b/nbt/snbt_decode.go @@ -242,7 +242,7 @@ func (d *decodeState) readIndex() int { // scanNext processes the byte at d.data[d.off]. func (d *decodeState) scanNext() { if d.off < len(d.data) { - d.opcode = d.scan.step(d.data[d.off]) + d.opcode = d.scan.step(&d.scan, d.data[d.off]) d.off++ } else { //d.opcode = d.scan.eof() @@ -255,7 +255,7 @@ func (d *decodeState) scanNext() { func (d *decodeState) scanWhile(op int) { s, data, i := &d.scan, d.data, d.off for i < len(data) { - newOp := s.step(data[i]) + newOp := s.step(s, data[i]) i++ if newOp != op { d.opcode = newOp diff --git a/nbt/snbt_scanner.go b/nbt/snbt_scanner.go index 44c043f..6f918eb 100644 --- a/nbt/snbt_scanner.go +++ b/nbt/snbt_scanner.go @@ -2,7 +2,6 @@ package nbt import ( "errors" - "sync" ) const ( @@ -34,7 +33,7 @@ const ( const maxNestingDepth = 10000 type scanner struct { - step func(c byte) int + step func(s *scanner, c byte) int parseState []int err error endTop bool @@ -43,30 +42,10 @@ type scanner struct { // reset prepares the scanner for use. // It must be called before calling s.step. func (s *scanner) reset() { - s.step = s.stateBeginValue + s.step = stateBeginValue s.parseState = s.parseState[0:0] -} - -var scannerPool = sync.Pool{ - New: func() interface{} { - return &scanner{} - }, -} - -func newScanner() *scanner { - scan := scannerPool.Get().(*scanner) - // scan.reset by design doesn't set bytes to zero - //scan.bytes = 0 - scan.reset() - return scan -} - -func freeScanner(scan *scanner) { - // Avoid hanging on to too much memory in extreme cases. - if len(scan.parseState) > 1024 { - scan.parseState = nil - } - scannerPool.Put(scan) + s.err = nil + s.endTop = false } // pushParseState pushes a new parse state p onto the parse stack. @@ -85,10 +64,10 @@ func (s *scanner) popParseState() { n := len(s.parseState) - 1 s.parseState = s.parseState[:n] if n == 0 { - s.step = s.stateEndTop + s.step = stateEndTop s.endTop = true } else { - s.step = s.stateEndValue + s.step = stateEndValue } } @@ -101,7 +80,7 @@ func (s *scanner) eof() int { if s.endTop { return scanEnd } - s.step(' ') + s.step(s, ' ') if s.endTop { return scanEnd } @@ -114,7 +93,7 @@ func (s *scanner) eof() int { // stateEndTop is the state after finishing the top-level value, // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. -func (s *scanner) stateEndTop(c byte) int { +func stateEndTop(s *scanner, c byte) int { if !isSpace(c) { // Complain about non-space byte on next call. s.error(c, "after top-level value") @@ -122,195 +101,195 @@ func (s *scanner) stateEndTop(c byte) int { return scanEnd } -func (s *scanner) stateBeginValue(c byte) int { +func stateBeginValue(s *scanner, c byte) int { if isSpace(c) { - s.step = s.stateBeginValue + s.step = stateBeginValue return scanSkipSpace } switch c { case '{': // beginning of TAG_Compound - s.step = s.stateCompoundOrEmpty + s.step = stateCompoundOrEmpty return s.pushParseState(c, parseCompoundName, scanBeginCompound) case '[': // beginning of TAG_List - s.step = s.stateListOrArray + s.step = stateListOrArray return s.pushParseState(c, parseListValue, scanBeginList) case '"', '\'': // beginning of TAG_String - return s.stateBeginString(c) + return stateBeginString(s, c) case '-': // beginning of negative number - s.step = s.stateNeg + s.step = stateNeg return scanBeginLiteral default: if isNumber(c) { - s.stateNum0(c) + stateNum0(s, c) return scanBeginLiteral } if isAllowedInUnquotedString(c) { - return s.stateBeginString(c) + return stateBeginString(s, c) } } return s.error(c, "looking for beginning of value") } -func (s *scanner) stateCompoundOrEmpty(c byte) int { +func stateCompoundOrEmpty(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } if c == '}' { n := len(s.parseState) s.parseState[n-1] = parseCompoundValue - return s.stateEndValue(c) + return stateEndValue(s, c) } - return s.stateBeginString(c) + return stateBeginString(s, c) } -func (s *scanner) stateBeginString(c byte) int { +func stateBeginString(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } switch c { case '\'': - s.step = s.stateInSingleQuotedString + s.step = stateInSingleQuotedString return scanBeginLiteral case '"': - s.step = s.stateInDoubleQuotedString + s.step = stateInDoubleQuotedString return scanBeginLiteral default: if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanBeginLiteral } } return s.error(c, "looking for beginning of string") } -func (s *scanner) stateInSingleQuotedString(c byte) int { +func stateInSingleQuotedString(s *scanner, c byte) int { if c == '\\' { - s.step = s.stateInSingleQuotedStringEsc + s.step = stateInSingleQuotedStringEsc return scanContinue } if c == '\'' { - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } return scanContinue } -func (s *scanner) stateInSingleQuotedStringEsc(c byte) int { +func stateInSingleQuotedStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': - s.step = s.stateInSingleQuotedString + s.step = stateInSingleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInDoubleQuotedString(c byte) int { +func stateInDoubleQuotedString(s *scanner, c byte) int { if c == '\\' { - s.step = s.stateInDqStringEsc + s.step = stateInDqStringEsc return scanContinue } if c == '"' { - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } return scanContinue } -func (s *scanner) stateInDqStringEsc(c byte) int { +func stateInDqStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': - s.step = s.stateInDoubleQuotedString + s.step = stateInDoubleQuotedString return scanContinue } return s.error(c, "in string escape code") } -func (s *scanner) stateInUnquotedString(c byte) int { +func stateInUnquotedString(s *scanner, c byte) int { if isAllowedInUnquotedString(c) { return scanContinue } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateListOrArray(c byte) int { +func stateListOrArray(s *scanner, c byte) int { if isSpace(c) { return scanSkipSpace } switch c { case 'B', 'I', 'L': - s.step = s.stateListOrArrayT + s.step = stateListOrArrayT return scanBeginLiteral case ']': - return s.stateEndValue(c) + return stateEndValue(s, c) default: - return s.stateBeginValue(c) + return stateBeginValue(s, c) } } -func (s *scanner) stateListOrArrayT(c byte) int { +func stateListOrArrayT(s *scanner, c byte) int { if c == ';' { - s.step = s.stateArrayT + s.step = stateArrayT return scanListType } - return s.stateInUnquotedString(c) + return stateInUnquotedString(s, c) } -func (s *scanner) stateArrayT(c byte) int { +func stateArrayT(s *scanner, c byte) int { if c == ']' { // empty array return scanEndValue } - return s.stateBeginValue(c) + return stateBeginValue(s, c) } -func (s *scanner) stateNeg(c byte) int { +func stateNeg(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum0 + s.step = stateNum0 return scanBeginLiteral } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanBeginLiteral } return s.error(c, "not a number after '-'") } -func (s *scanner) stateNum0(c byte) int { +func stateNum0(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum1 + s.step = stateNum1 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumValue(c) + return stateEndNumValue(s, c) } -func (s *scanner) stateNum1(c byte) int { +func stateNum1(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNum1 + s.step = stateNum1 return scanContinue } if c == '.' { - s.step = s.stateNumDot + s.step = stateNumDot return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumValue(c) + return stateEndNumValue(s, c) } // stateDot is the state after reading the integer and decimal point in a number, // such as after reading `1.`. -func (s *scanner) stateNumDot(c byte) int { +func stateNumDot(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNumDot0 + s.step = stateNumDot0 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } return s.error(c, "after decimal point in numeric literal") @@ -318,54 +297,54 @@ func (s *scanner) stateNumDot(c byte) int { // stateNumDot0 is the state after reading the integer, decimal point, and subsequent // digits of a number, such as after reading `3.14`. -func (s *scanner) stateNumDot0(c byte) int { +func stateNumDot0(s *scanner, c byte) int { if isNumber(c) { - s.step = s.stateNumDot0 + s.step = stateNumDot0 return scanContinue } if isAllowedInUnquotedString(c) { - s.step = s.stateInUnquotedString + s.step = stateInUnquotedString return scanContinue } - return s.stateEndNumDotValue(c) + return stateEndNumDotValue(s, c) } -func (s *scanner) stateEndNumValue(c byte) int { +func stateEndNumValue(s *scanner, c byte) int { switch c { case 'b', 'B': // TAG_Byte - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 's', 'S': // TAG_Short - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'l', 'L': // TAG_Long - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'f', 'F', 'd', 'D': - return s.stateEndNumDotValue(c) + return stateEndNumDotValue(s, c) } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateEndNumDotValue(c byte) int { +func stateEndNumDotValue(s *scanner, c byte) int { switch c { case 'f', 'F': // TAG_Float - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue case 'd', 'D': // TAG_Double - s.step = s.stateEndValue + s.step = stateEndValue return scanContinue } - return s.stateEndValue(c) + return stateEndValue(s, c) } -func (s *scanner) stateEndValue(c byte) int { +func stateEndValue(s *scanner, c byte) int { n := len(s.parseState) if n == 0 { // Completed top-level before the current byte. - s.step = s.stateEndTop + s.step = stateEndTop s.endTop = true - return s.stateEndTop(c) + return stateEndTop(s, c) } if isSpace(c) { return scanSkipSpace @@ -376,7 +355,7 @@ func (s *scanner) stateEndValue(c byte) int { case parseCompoundName: if c == ':' { s.parseState[n-1] = parseCompoundValue - s.step = s.stateBeginValue + s.step = stateBeginValue return scanCompoundTagName } return s.error(c, "after compound tag name") @@ -384,7 +363,7 @@ func (s *scanner) stateEndValue(c byte) int { switch c { case ',': s.parseState[n-1] = parseCompoundName - s.step = s.stateBeginString + s.step = stateBeginString return scanCompoundValue case '}': s.popParseState() @@ -394,7 +373,7 @@ func (s *scanner) stateEndValue(c byte) int { case parseListValue: switch c { case ',': - s.step = s.stateBeginValue + s.step = stateBeginValue return scanListValue case ']': s.popParseState() @@ -406,14 +385,14 @@ func (s *scanner) stateEndValue(c byte) int { } func (s *scanner) error(c byte, context string) int { - s.step = s.stateError + s.step = stateError s.err = errors.New(context) return scanError } // stateError is the state after reaching a syntax error, // such as after reading `[1}` or `5.1.2`. -func (s *scanner) stateError(c byte) int { +func stateError(s *scanner, c byte) int { return scanError } diff --git a/nbt/snbt_scanner_test.go b/nbt/snbt_scanner_test.go index cf5de10..b22d520 100644 --- a/nbt/snbt_scanner_test.go +++ b/nbt/snbt_scanner_test.go @@ -9,8 +9,8 @@ func TestSNBT_checkScanCode(t *testing.T) { //t.SkipNow() var s scanner s.reset() - for _, c := range []byte(`[{},{a:1b},{}]`) { - t.Logf("[%c] - %d", c, s.step(c)) + for _, c := range []byte(`[I;123,345]`) { + t.Logf("[%c] - %d", c, s.step(&s, c)) } t.Logf("[%c] - %d", ' ', s.eof()) } @@ -26,7 +26,7 @@ func TestSNBT_number(t *testing.T) { scan := func(str string) bool { s.reset() for _, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { return false } @@ -53,7 +53,7 @@ func TestSNBT_compound(t *testing.T) { for _, str := range goods { s.reset() for i, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { t.Errorf("scan valid data %q error: %v at [%d]", str[:i], s.err, i) break @@ -73,7 +73,7 @@ func TestSNBT_list(t *testing.T) { scan := func(str string) bool { s.reset() for _, c := range []byte(str) { - res := s.step(c) + res := s.step(&s, c) if res == scanError { return false } @@ -86,3 +86,17 @@ func TestSNBT_list(t *testing.T) { } } } + +func BenchmarkSNBT_bigTest(b *testing.B) { + var s scanner + for i := 0; i < b.N; i++ { + s.reset() + for _, c := range []byte(bigTest) { + res := s.step(&s, c) + if res == scanError { + b.Errorf("scan valid data %q error: %v at [%d]", bigTest[:i], s.err, i) + break + } + } + } +}