proc/gdbserial: fix two protocol bugs (#2172)

During the testing of the core dump generation feature two bugs were
discovered in gdbserial:

1. we don't check that both bytes of the checksum are read, if the
   buffer only has one byte we can end up reading only one byte instead
   of two and the second byte will mess up the parsing of the next
   packet
2. binary encoded packets can start with an 'E' and not be errors, when
   using binary responses add an extra check for the lenght of the
   response before deciding that the response is an error.
   Unfortunately this encoding is inherently ambiguous (we can't
   distinguish a 3 byte response starting with 'E' from an error) so
   binary requests that lead to short responses should be avoided.

Testing this is complicated, they will be tested implicitly by the
upcoming core dump test.
Co-authored-by: Na <a@kra>

pkg/proc/gdbserial/gdbserver_conn.go

@@ -1084,14 +1084,14 @@ func (conn *gdbConn) recv(cmd []byte, context string, binary bool) (resp []byte,
 		}
 
 		// read checksum
-		_, err = conn.rdr.Read(conn.inbuf[:2])
+		_, err = io.ReadFull(conn.rdr, conn.inbuf[:2])
 		if err != nil {
 			return nil, err
 		}
 		if logflags.GdbWire() {
 			out := resp
 			partial := false
-			if idx := bytes.Index(out, []byte{'\n'}); idx >= 0 {
+			if idx := bytes.Index(out, []byte{'\n'}); idx >= 0 && !binary {
 				out = resp[:idx]
 				partial = true
 			}
@@ -1100,9 +1100,17 @@ func (conn *gdbConn) recv(cmd []byte, context string, binary bool) (resp []byte,
 				partial = true
 			}
 			if !partial {
-				conn.log.Debugf("-> %s%s", string(resp), string(conn.inbuf[:2]))
+				if binary {
+					conn.log.Debugf("-> %q%s", string(resp), string(conn.inbuf[:2]))
+				} else {
+					conn.log.Debugf("-> %s%s", string(resp), string(conn.inbuf[:2]))
+				}
 			} else {
-				conn.log.Debugf("-> %s...", string(out))
+				if binary {
+					conn.log.Debugf("-> %q...", string(out))
+				} else {
+					conn.log.Debugf("-> %s...", string(out))
+				}
 			}
 		}
 
@@ -1136,7 +1144,7 @@ func (conn *gdbConn) recv(cmd []byte, context string, binary bool) (resp []byte,
 		conn.inbuf, resp = wiredecode(resp, conn.inbuf)
 	}
 
-	if len(resp) == 0 || resp[0] == 'E' {
+	if len(resp) == 0 || (resp[0] == 'E' && !binary) || (resp[0] == 'E' && len(resp) == 3) {
 		cmdstr := ""
 		if cmd != nil {
 			cmdstr = string(cmd)