proc/gdbserial: mozilla rr support (#804)

Implements #727

Documentation/cli/README.md

@@ -5,7 +5,10 @@ Command | Description
 [args](#args) | Print function arguments.
 [break](#break) | Sets a breakpoint.
 [breakpoints](#breakpoints) | Print out info for active breakpoints.
+[check](#check) | Creates a checkpoint at the current position.
+[checkpoints](#checkpoints) | Print out info for existing checkpoints.
 [clear](#clear) | Deletes breakpoint.
+[clear-checkpoint](#clear-checkpoint) | Deletes checkpoint.
 [clearall](#clearall) | Deletes multiple breakpoints.
 [condition](#condition) | Set breakpoint condition.
 [continue](#continue) | Run until breakpoint or program termination.
@@ -22,7 +25,8 @@ Command | Description
 [on](#on) | Executes a command when a breakpoint is hit.
 [print](#print) | Evaluate an expression.
 [regs](#regs) | Print contents of CPU registers.
-[restart](#restart) | Restart process.
+[restart](#restart) | Restart process from a checkpoint or event.
+[rewind](#rewind) | Run backwards until breakpoint or program termination.
 [set](#set) | Changes the value of a variable.
 [source](#source) | Executes a file containing a list of delve commands
 [sources](#sources) | Print list of source files.
@@ -60,12 +64,30 @@ Print out info for active breakpoints.
 
 Aliases: bp
 
+## check
+Creates a checkpoint at the current position.
+			
+	checkpoint [where]
+
+Aliases: checkpoint
+
+## checkpoints
+Print out info for existing checkpoints.
+
+
 ## clear
 Deletes breakpoint.
 
 	clear <breakpoint name or id>
 
 
+## clear-checkpoint
+Deletes checkpoint.
+			
+	checkpoint <id>
+
+Aliases: clearcheck
+
 ## clearall
 Deletes multiple breakpoints.
 
@@ -202,10 +224,17 @@ Argument -a shows more registers.
 
 
 ## restart
-Restart process.
+Restart process from a checkpoint or event.
+	
+	restart [event number or checkpoint id]
 
 Aliases: r
 
+## rewind
+Run backwards until breakpoint or program termination.
+
+Aliases: rw
+
 ## set
 Changes the value of a variable.
 

Documentation/usage/dlv.md

@@ -19,24 +19,32 @@ Pass flags to the program you are debugging using `--`, for example:
 ### Options
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv attach](dlv_attach.md)	 - Attach to running process and begin debugging.
 * [dlv connect](dlv_connect.md)	 - Connect to a headless debug server.
+* [dlv core](dlv_core.md)	 - Examine a core dump.
 * [dlv debug](dlv_debug.md)	 - Compile and begin debugging main package in current directory, or the package specified.
 * [dlv exec](dlv_exec.md)	 - Execute a precompiled binary, and begin a debug session.
+* [dlv replay](dlv_replay.md)	 - Replays a rr trace.
 * [dlv run](dlv_run.md)	 - Deprecated command. Use 'debug' instead.
 * [dlv test](dlv_test.md)	 - Compile test binary and begin debugging program.
 * [dlv trace](dlv_trace.md)	 - Compile and begin tracing program.
 * [dlv version](dlv_version.md)	 - Prints version.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_attach.md

@@ -13,23 +13,29 @@ option to let the process continue or kill it.
 
 
 ```
-dlv attach pid
+dlv attach pid [executable]
 ```
 
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_connect.md

@@ -14,17 +14,23 @@ dlv connect addr
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_core.md

+## dlv core
+
+Examine a core dump.
+
+### Synopsis
+
+
+Examine a core dump.
+		
+The core command will open the specified core file and the associated
+executable and let you examine the state of the process when the
+core dump was taken.
+
+```
+dlv core <executable> <core>
+```
+
+### Options inherited from parent commands
+
+```
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
+```
+
+### SEE ALSO
+* [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
+
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_debug.md

@@ -19,17 +19,23 @@ dlv debug [package]
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_exec.md

@@ -19,17 +19,23 @@ dlv exec [./path/to/binary]
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_replay.md

+## dlv replay
+
+Replays a rr trace.
+
+### Synopsis
+
+
+Replays a rr trace.
+			
+The replay command will open a trace generated by mozilla rr. Mozilla rr must be installed:
+https://github.com/mozilla/rr
+			
+
+```
+dlv replay [trace directory]
+```
+
+### Options inherited from parent commands
+
+```
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
+```
+
+### SEE ALSO
+* [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
+
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_run.md

@@ -14,17 +14,23 @@ dlv run
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_test.md

@@ -19,17 +19,23 @@ dlv test [package]
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_trace.md

@@ -19,24 +19,30 @@ dlv trace [package] regexp
 ### Options
 
 ```
-  -p, --pid=0: Pid to attach to.
-  -s, --stack=0: Show stack trace with given depth.
+  -p, --pid int     Pid to attach to.
+  -s, --stack int   Show stack trace with given depth.
 ```
 
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Documentation/usage/dlv_version.md

@@ -14,17 +14,23 @@ dlv version
 ### Options inherited from parent commands
 
 ```
-      --accept-multiclient[=false]: Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
-      --api-version=1: Selects API version when headless.
-      --build-flags="": Build flags, to be passed to the compiler.
-      --headless[=false]: Run debug server only, in headless mode.
-      --init="": Init file, executed by the terminal client.
-  -l, --listen="localhost:0": Debugging server listen address.
-      --log[=false]: Enable debugging server logging.
-      --wd=".": Working directory for running the program.
+      --accept-multiclient   Allows a headless server to accept multiple client connections. Note that the server API is not reentrant and clients will have to coordinate.
+      --api-version int      Selects API version when headless. (default 1)
+      --backend string       Backend selection:
+	default		Uses lldb on macOS, native everywhere else.
+	native		Native backend.
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+ (default "default")
+      --build-flags string   Build flags, to be passed to the compiler.
+      --headless             Run debug server only, in headless mode.
+      --init string          Init file, executed by the terminal client.
+  -l, --listen string        Debugging server listen address. (default "localhost:0")
+      --log                  Enable debugging server logging.
+      --wd string            Working directory for running the program. (default ".")
 ```
 
 ### SEE ALSO
 * [dlv](dlv.md)	 - Delve is a debugger for the Go programming language.
 
-###### Auto generated by spf13/cobra on 15-Feb-2017
+###### Auto generated by spf13/cobra on 5-May-2017

Makefile

@@ -73,6 +73,17 @@ ifneq "$(shell which lldb-server)" ""
 	@echo 'Testing LLDB backend (terminal)'
 	go test $(TEST_FLAGS) $(BUILD_FLAGS) $(PREFIX)/pkg/terminal -backend=lldb
 endif
+ifneq "$(shell which rr)" ""
+	@echo
+	@echo 'Testing Mozilla RR backend (proc)'
+	go test $(TEST_FLAGS) $(BUILD_FLAGS) $(PREFIX)/pkg/proc -backend=rr
+	@echo
+	@echo 'Testing Mozilla RR backend (integration)'
+	go test $(TEST_FLAGS) $(BUILD_FLAGS) $(PREFIX)/service/test -backend=rr
+	@echo
+	@echo 'Testing Mozilla RR backend (terminal)'
+	go test $(TEST_FLAGS) $(BUILD_FLAGS) $(PREFIX)/pkg/terminal -backend=rr
+endif
 
 test-proc-run:
 	go test $(TEST_FLAGS) $(BUILD_FLAGS) -test.v -test.run="$(RUN)" -backend=$(BACKEND) $(PREFIX)/pkg/proc

cmd/dlv/cmds/commands.go

@@ -98,7 +98,9 @@ func New() *cobra.Command {
 	RootCommand.PersistentFlags().StringVar(&Backend, "backend", "default", `Backend selection:
 	default		Uses lldb on macOS, native everywhere else.
 	native		Native backend.
-	lldb		Uses lldb-server or debugserver.`)
+	lldb		Uses lldb-server or debugserver.
+	rr		Uses mozilla rr (https://github.com/mozilla/rr).
+`)
 
 	// 'attach' subcommand.
 	attachCommand := &cobra.Command{
@@ -240,6 +242,29 @@ core dump was taken.`,
 	}
 	RootCommand.AddCommand(versionCommand)
 
+	if path, _ := exec.LookPath("rr"); path != "" {
+		replayCommand := &cobra.Command{
+			Use:   "replay [trace directory]",
+			Short: "Replays a rr trace.",
+			Long: `Replays a rr trace.
+			
+The replay command will open a trace generated by mozilla rr. Mozilla rr must be installed:
+https://github.com/mozilla/rr
+			`,
+			PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+				if len(args) == 0 {
+					return errors.New("you must provide a path to a binary")
+				}
+				return nil
+			},
+			Run: func(cmd *cobra.Command, args []string) {
+				Backend = "rr"
+				os.Exit(execute(0, []string{}, conf, args[0], executingOther))
+			},
+		}
+		RootCommand.AddCommand(replayCommand)
+	}
+
 	return RootCommand
 }
 

pkg/proc/core/core.go

@@ -194,6 +194,14 @@ func (p *Process) BinInfo() *proc.BinaryInfo {
 	return &p.bi
 }
 
+func (p *Process) Recorded() (bool, string)                { return true, "" }
+func (p *Process) Restart(string) error                    { return ErrContinueCore }
+func (p *Process) Direction(proc.Direction) error          { return ErrContinueCore }
+func (p *Process) When() (string, error)                   { return "", nil }
+func (p *Process) Checkpoint(string) (int, error)          { return -1, ErrContinueCore }
+func (p *Process) Checkpoints() ([]proc.Checkpoint, error) { return nil, nil }
+func (p *Process) ClearCheckpoint(int) error               { return errors.New("checkpoint not found") }
+
 func (thread *Thread) ReadMemory(data []byte, addr uintptr) (n int, err error) {
 	n, err = thread.p.core.ReadMemory(data, addr)
 	if err == nil && n != len(data) {

pkg/proc/gdbserial/gdbserver.go

@@ -92,6 +92,8 @@ const (
 
 const heartbeatInterval = 10 * time.Second
 
+var ErrDirChange = errors.New("direction change with internal breakpoints")
+
 // Process implements proc.Process using a connection to a debugger stub
 // that understands Gdb Remote Serial Protocol.
 type Process struct {
@@ -109,8 +111,9 @@ type Process struct {
 	breakpointIDCounter         int
 	internalBreakpointIDCounter int
 
-	gcmdok         bool // true if the stub supports g and G commands
-	threadStopInfo bool // true if the stub supports qThreadStopInfo
+	gcmdok         bool   // true if the stub supports g and G commands
+	threadStopInfo bool   // true if the stub supports qThreadStopInfo
+	tracedir       string // if attached to rr the path to the trace directory
 
 	loadGInstrAddr uint64 // address of the g loading instruction, zero if we couldn't allocate it
 
@@ -173,6 +176,7 @@ func Connect(addr string, path string, pid int, attempts int) (*Process, error)
 			conn:                conn,
 			maxTransmitAttempts: maxTransmitAttempts,
 			inbuf:               make([]byte, 0, initialInputBufferSize),
+			direction:           proc.Forward,
 		},
 
 		threads:        make(map[int]*Thread),
@@ -241,7 +245,7 @@ func Connect(addr string, path string, pid int, attempts int) (*Process, error)
 
 	if p.conn.pid <= 0 {
 		p.conn.pid, _, err = p.loadProcessInfo(0)
-		if err != nil {
+		if err != nil && !isProtocolErrorUnsupported(err) {
 			conn.Close()
 			p.bi.Close()
 			return nil, err
@@ -415,6 +419,10 @@ func (p *Process) BinInfo() *proc.BinaryInfo {
 	return &p.bi
 }
 
+func (p *Process) Recorded() (bool, string) {
+	return p.tracedir != "", p.tracedir
+}
+
 func (p *Process) Pid() int {
 	return int(p.conn.pid)
 }
@@ -464,11 +472,13 @@ func (p *Process) ContinueOnce() (proc.Thread, error) {
 		return nil, &proc.ProcessExitedError{Pid: p.conn.pid}
 	}
 
-	// step threads stopped at any breakpoint over their breakpoint
-	for _, thread := range p.threads {
-		if thread.CurrentBreakpoint != nil {
-			if err := thread.stepInstruction(&threadUpdater{p: p}); err != nil {
-				return nil, err
+	if p.conn.direction == proc.Forward {
+		// step threads stopped at any breakpoint over their breakpoint
+		for _, thread := range p.threads {
+			if thread.CurrentBreakpoint != nil {
+				if err := thread.stepInstruction(&threadUpdater{p: p}); err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
@@ -593,11 +603,17 @@ func (p *Process) SwitchGoroutine(gid int) error {
 }
 
 func (p *Process) RequestManualStop() error {
+	if !p.conn.running {
+		return nil
+	}
 	p.ctrlC = true
 	return p.conn.sendCtrlC()
 }
 
 func (p *Process) Halt() error {
+	if p.exited {
+		return nil
+	}
 	p.ctrlC = true
 	return p.conn.sendCtrlC()
 }
@@ -634,6 +650,153 @@ func (p *Process) Detach(kill bool) error {
 	return p.bi.Close()
 }
 
+func (p *Process) Restart(pos string) error {
+	if p.tracedir == "" {
+		return proc.NotRecordedErr
+	}
+
+	p.exited = false
+
+	p.allGCache = nil
+	for _, th := range p.threads {
+		th.clearBreakpointState()
+	}
+
+	p.ctrlC = false
+
+	err := p.conn.restart(pos)
+	if err != nil {
+		return err
+	}
+
+	// for some reason we have to send a vCont;c after a vRun to make rr behave
+	// properly, because that's what gdb does.
+	_, _, err = p.conn.resume(0, nil)
+	if err != nil {
+		return err
+	}
+
+	err = p.updateThreadList(&threadUpdater{p: p})
+	if err != nil {
+		return err
+	}
+	p.selectedGoroutine, _ = proc.GetG(p.CurrentThread())
+
+	for addr := range p.breakpoints {
+		p.conn.setBreakpoint(addr)
+	}
+
+	if err := p.setCurrentBreakpoints(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Process) When() (string, error) {
+	if p.tracedir == "" {
+		return "", proc.NotRecordedErr
+	}
+	event, err := p.conn.qRRCmd("when")
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(event), nil
+}
+
+const (
+	checkpointPrefix = "Checkpoint "
+)
+
+func (p *Process) Checkpoint(where string) (int, error) {
+	if p.tracedir == "" {
+		return -1, proc.NotRecordedErr
+	}
+	resp, err := p.conn.qRRCmd("checkpoint", where)
+	if err != nil {
+		return -1, err
+	}
+
+	if !strings.HasPrefix(resp, checkpointPrefix) {
+		return -1, fmt.Errorf("can not parse checkpoint response %q", resp)
+	}
+
+	idstr := resp[len(checkpointPrefix):]
+	space := strings.Index(idstr, " ")
+	if space < 0 {
+		return -1, fmt.Errorf("can not parse checkpoint response %q", resp)
+	}
+	idstr = idstr[:space]
+
+	cpid, err := strconv.Atoi(idstr)
+	if err != nil {
+		return -1, err
+	}
+	return cpid, nil
+}
+
+func (p *Process) Checkpoints() ([]proc.Checkpoint, error) {
+	if p.tracedir == "" {
+		return nil, proc.NotRecordedErr
+	}
+	resp, err := p.conn.qRRCmd("info checkpoints")
+	if err != nil {
+		return nil, err
+	}
+	lines := strings.Split(resp, "\n")
+	r := make([]proc.Checkpoint, 0, len(lines)-1)
+	for _, line := range lines[1:] {
+		if line == "" {
+			continue
+		}
+		fields := strings.Split(line, "\t")
+		if len(fields) != 3 {
+			return nil, fmt.Errorf("can not parse \"info checkpoints\" output line %q", line)
+		}
+		cpid, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return nil, fmt.Errorf("can not parse \"info checkpoints\" output line %q: %v", line, err)
+		}
+		r = append(r, proc.Checkpoint{cpid, fields[1], fields[2]})
+	}
+	return r, nil
+}
+
+const deleteCheckpointPrefix = "Deleted checkpoint "
+
+func (p *Process) ClearCheckpoint(id int) error {
+	if p.tracedir == "" {
+		return proc.NotRecordedErr
+	}
+	resp, err := p.conn.qRRCmd("delete checkpoint", strconv.Itoa(id))
+	if err != nil {
+		return err
+	}
+	if !strings.HasPrefix(resp, deleteCheckpointPrefix) {
+		return errors.New(resp)
+	}
+	return nil
+}
+
+func (p *Process) Direction(dir proc.Direction) error {
+	if p.tracedir == "" {
+		return proc.NotRecordedErr
+	}
+	if p.conn.conn == nil {
+		return proc.ProcessExitedError{Pid: p.conn.pid}
+	}
+	if p.conn.direction == dir {
+		return nil
+	}
+	for _, bp := range p.Breakpoints() {
+		if bp.Internal() {
+			return ErrDirChange
+		}
+	}
+	p.conn.direction = dir
+	return nil
+}
+
 func (p *Process) Breakpoints() map[uint64]*proc.Breakpoint {
 	return p.breakpoints
 }
@@ -762,6 +925,12 @@ func (tu *threadUpdater) Finish() {
 			tu.p.currentThread = nil
 		}
 	}
+	if tu.p.currentThread != nil {
+		if _, exists := tu.p.threads[tu.p.currentThread.ID]; !exists {
+			// current thread was removed
+			tu.p.currentThread = nil
+		}
+	}
 	if tu.p.currentThread == nil {
 		for _, thread := range tu.p.threads {
 			tu.p.currentThread = thread
@@ -991,6 +1160,16 @@ func (t *Thread) reloadRegisters() error {
 		}
 	}
 
+	switch t.p.bi.GOOS {
+	case "linux":
+		if reg, hasFsBase := t.regs.regs[regnameFsBase]; hasFsBase {
+			t.regs.gaddr = 0
+			t.regs.tls = binary.LittleEndian.Uint64(reg.value)
+			t.regs.hasgaddr = false
+			return nil
+		}
+	}
+
 	if t.p.loadGInstrAddr > 0 {
 		return t.reloadGAlloc()
 	}

pkg/proc/gdbserial/gdbserver_conn.go

@@ -25,6 +25,8 @@ type gdbConn struct {
 
 	running bool
 
+	direction proc.Direction // direction of execution
+
 	packetSize int               // maximum packet size supported by stub
 	regsInfo   []gdbRegisterInfo // list of registers
 
@@ -38,10 +40,12 @@ type gdbConn struct {
 }
 
 const (
-	regnamePC = "rip"
-	regnameCX = "rcx"
-	regnameSP = "rsp"
-	regnameBP = "rbp"
+	regnamePC     = "rip"
+	regnameCX     = "rcx"
+	regnameSP     = "rsp"
+	regnameBP     = "rbp"
+	regnameFsBase = "fs_base"
+	regnameGsBase = "gs_base"
 )
 
 var ErrTooManyAttempts = errors.New("too many transmit attempts")
@@ -269,6 +273,9 @@ func (conn *gdbConn) readRegisterInfo() (err error) {
 		fmt.Fprintf(&conn.outbuf, "$qRegisterInfo%x", regnum)
 		respbytes, err := conn.exec(conn.outbuf.Bytes(), "register info")
 		if err != nil {
+			if regnum == 0 {
+				return err
+			}
 			break
 		}
 
@@ -410,7 +417,7 @@ func (conn *gdbConn) kill() error {
 		// kill. This is not an error.
 		conn.conn.Close()
 		conn.conn = nil
-		return nil
+		return proc.ProcessExitedError{Pid: conn.pid}
 	}
 	if err != nil {
 		return err
@@ -515,11 +522,19 @@ func (conn *gdbConn) writeRegister(threadID string, regnum int, data []byte) err
 // resume executes a 'vCont' command on all threads with action 'c' if sig
 // is 0 or 'C' if it isn't.
 func (conn *gdbConn) resume(sig uint8, tu *threadUpdater) (string, uint8, error) {
-	conn.outbuf.Reset()
-	if sig == 0 {
-		fmt.Fprintf(&conn.outbuf, "$vCont;c")
+	if conn.direction == proc.Forward {
+		conn.outbuf.Reset()
+		if sig == 0 {
+			fmt.Fprintf(&conn.outbuf, "$vCont;c")
+		} else {
+			fmt.Fprintf(&conn.outbuf, "$vCont;C%02x", sig)
+		}
 	} else {
-		fmt.Fprintf(&conn.outbuf, "$vCont;C%02x", sig)
+		if err := conn.selectThread('c', "p-1.-1", "resume"); err != nil {
+			return "", 0, err
+		}
+		conn.outbuf.Reset()
+		fmt.Fprintf(&conn.outbuf, "$bc")
 	}
 	if err := conn.send(conn.outbuf.Bytes()); err != nil {
 		return "", 0, err
@@ -533,8 +548,16 @@ func (conn *gdbConn) resume(sig uint8, tu *threadUpdater) (string, uint8, error)
 
 // step executes a 'vCont' command on the specified thread with 's' action.
 func (conn *gdbConn) step(threadID string, tu *threadUpdater) (string, uint8, error) {
-	conn.outbuf.Reset()
-	fmt.Fprintf(&conn.outbuf, "$vCont;s:%s", threadID)
+	if conn.direction == proc.Forward {
+		conn.outbuf.Reset()
+		fmt.Fprintf(&conn.outbuf, "$vCont;s:%s", threadID)
+	} else {
+		if err := conn.selectThread('c', threadID, "step"); err != nil {
+			return "", 0, err
+		}
+		conn.outbuf.Reset()
+		fmt.Fprintf(&conn.outbuf, "$bs")
+	}
 	if err := conn.send(conn.outbuf.Bytes()); err != nil {
 		return "", 0, err
 	}
@@ -808,15 +831,19 @@ func (conn *gdbConn) readMemory(data []byte, addr uintptr) error {
 	return nil
 }
 
+func writeAsciiBytes(w io.Writer, data []byte) {
+	for _, b := range data {
+		fmt.Fprintf(w, "%02x", b)
+	}
+}
+
 // executes 'M' (write memory) command
 func (conn *gdbConn) writeMemory(addr uintptr, data []byte) (written int, err error) {
 	conn.outbuf.Reset()
 	//TODO(aarzilli): do not send packets larger than conn.PacketSize
 	fmt.Fprintf(&conn.outbuf, "$M%x,%x:", addr, len(data))
 
-	for _, b := range data {
-		fmt.Fprintf(&conn.outbuf, "%02x", b)
-	}
+	writeAsciiBytes(&conn.outbuf, data)
 
 	_, err = conn.exec(conn.outbuf.Bytes(), "memory write")
 	if err != nil {
@@ -851,6 +878,41 @@ func (conn *gdbConn) threadStopInfo(threadID string) (sig uint8, reason string,
 	return sp.sig, sp.reason, nil
 }
 
+// restart executes a 'vRun' command.
+func (conn *gdbConn) restart(pos string) error {
+	conn.outbuf.Reset()
+	fmt.Fprintf(&conn.outbuf, "$vRun;")
+	if pos != "" {
+		fmt.Fprintf(&conn.outbuf, ";")
+		writeAsciiBytes(&conn.outbuf, []byte(pos))
+	}
+	_, err := conn.exec(conn.outbuf.Bytes(), "restart")
+	return err
+}
+
+// qRRCmd executes a qRRCmd command
+func (conn *gdbConn) qRRCmd(args ...string) (string, error) {
+	if len(args) == 0 {
+		panic("must specify at least one argument for qRRCmd")
+	}
+	conn.outbuf.Reset()
+	fmt.Fprintf(&conn.outbuf, "$qRRCmd")
+	for _, arg := range args {
+		fmt.Fprintf(&conn.outbuf, ":")
+		writeAsciiBytes(&conn.outbuf, []byte(arg))
+	}
+	resp, err := conn.exec(conn.outbuf.Bytes(), "qRRCmd")
+	if err != nil {
+		return "", err
+	}
+	data := make([]byte, 0, len(resp)/2)
+	for i := 0; i < len(resp); i += 2 {
+		n, _ := strconv.ParseUint(string(resp[i:i+2]), 16, 8)
+		data = append(data, uint8(n))
+	}
+	return string(data), nil
+}
+
 // exec executes a message to the stub and reads a response.
 // The details of the wire protocol are described here:
 //  https://sourceware.org/gdb/onlinedocs/gdb/Overview.html#Overview

pkg/proc/gdbserial/rr.go

+package gdbserial
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"unicode"
+)
+
+// Record uses rr to record the execution of the specified program and
+// returns the trace directory's path.
+func Record(cmd []string, wd string, quiet bool) (tracedir string, err error) {
+	rfd, wfd, err := os.Pipe()
+	if err != nil {
+		return "", err
+	}
+
+	args := make([]string, 0, len(cmd)+2)
+	args = append(args, "record", "--print-trace-dir=3")
+	args = append(args, cmd...)
+	rrcmd := exec.Command("rr", args...)
+	rrcmd.Stdin = os.Stdin
+	if !quiet {
+		rrcmd.Stdout = os.Stdout
+		rrcmd.Stderr = os.Stderr
+	}
+	rrcmd.ExtraFiles = []*os.File{wfd}
+	rrcmd.Dir = wd
+
+	done := make(chan struct{})
+	go func() {
+		bs, _ := ioutil.ReadAll(rfd)
+		tracedir = strings.TrimSpace(string(bs))
+		close(done)
+	}()
+
+	err = rrcmd.Run()
+	// ignore run errors, it could be the program crashing
+	wfd.Close()
+	<-done
+	return
+}
+
+// Replay starts an instance of rr in replay mode, with the specified trace
+// directory, and connects to it.
+func Replay(tracedir string, quiet bool) (*Process, error) {
+	rrcmd := exec.Command("rr", "replay", "--dbgport=0", tracedir)
+	rrcmd.Stdout = os.Stdout
+	stderr, err := rrcmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+	rrcmd.SysProcAttr = backgroundSysProcAttr()
+
+	initch := make(chan rrInit)
+	go rrStderrParser(stderr, initch, quiet)
+
+	err = rrcmd.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	init := <-initch
+	if init.err != nil {
+		rrcmd.Process.Kill()
+		return nil, err
+	}
+	p, err := Connect(init.port, init.exe, 0, 10)
+	if err != nil {
+		rrcmd.Process.Kill()
+		return nil, err
+	}
+
+	p.process = rrcmd
+	p.tracedir = tracedir
+
+	return p, nil
+}
+
+type rrInit struct {
+	port string
+	exe  string
+	err  error
+}
+
+const (
+	rrGdbCommandPrefix = "  gdb "
+	rrGdbLaunchPrefix  = "Launch gdb with"
+	targetCmd          = "target extended-remote "
+)
+
+func rrStderrParser(stderr io.Reader, initch chan<- rrInit, quiet bool) {
+	rd := bufio.NewReader(stderr)
+	for {
+		line, err := rd.ReadString('\n')
+		if err != nil {
+			initch <- rrInit{"", "", err}
+			close(initch)
+			return
+		}
+
+		if strings.HasPrefix(line, rrGdbCommandPrefix) {
+			initch <- rrParseGdbCommand(line[len(rrGdbCommandPrefix):])
+			close(initch)
+			break
+		}
+
+		if strings.HasPrefix(line, rrGdbLaunchPrefix) {
+			continue
+		}
+
+		if !quiet {
+			os.Stderr.Write([]byte(line))
+		}
+	}
+
+	io.Copy(os.Stderr, rd)
+}
+
+type ErrMalformedRRGdbCommand struct {
+	line, reason string
+}
+
+func (err *ErrMalformedRRGdbCommand) Error() string {
+	return fmt.Sprintf("malformed gdb command %q: %s", err.line, err.reason)
+}
+
+func rrParseGdbCommand(line string) rrInit {
+	port := ""
+	fields := splitQuotedFields(line)
+	for i := 0; i < len(fields); i++ {
+		switch fields[i] {
+		case "-ex":
+			if i+1 >= len(fields) {
+				return rrInit{err: &ErrMalformedRRGdbCommand{line, "-ex not followed by an argument"}}
+			}
+			arg := fields[i+1]
+
+			if !strings.HasPrefix(arg, targetCmd) {
+				return rrInit{err: &ErrMalformedRRGdbCommand{line, "contents of -ex argument unexpected"}}
+			}
+
+			port = arg[len(targetCmd):]
+			i++
+
+		case "-l":
+			// skip argument
+			i++
+		}
+	}
+
+	if port == "" {
+		return rrInit{err: &ErrMalformedRRGdbCommand{line, "could not find -ex argument"}}
+	}
+
+	exe := fields[len(fields)-1]
+
+	return rrInit{port: port, exe: exe}
+}
+
+// Like strings.Fields but ignores spaces inside areas surrounded
+// by single quotes.
+// To specify a single quote use backslash to escape it: '\''
+func splitQuotedFields(in string) []string {
+	type stateEnum int
+	const (
+		inSpace stateEnum = iota
+		inField
+		inQuote
+		inQuoteEscaped
+	)
+	state := inSpace
+	r := []string{}
+	var buf bytes.Buffer
+
+	for _, ch := range in {
+		switch state {
+		case inSpace:
+			if ch == '\'' {
+				state = inQuote
+			} else if !unicode.IsSpace(ch) {
+				buf.WriteRune(ch)
+				state = inField
+			}
+
+		case inField:
+			if ch == '\'' {
+				state = inQuote
+			} else if unicode.IsSpace(ch) {
+				r = append(r, buf.String())
+				buf.Reset()
+			} else {
+				buf.WriteRune(ch)
+			}
+
+		case inQuote:
+			if ch == '\'' {
+				state = inField
+			} else if ch == '\\' {
+				state = inQuoteEscaped
+			} else {
+				buf.WriteRune(ch)
+			}
+
+		case inQuoteEscaped:
+			buf.WriteRune(ch)
+			state = inQuote
+		}
+	}
+
+	if buf.Len() != 0 {
+		r = append(r, buf.String())
+	}
+
+	return r
+}
+
+// RecordAndReplay acts like calling Record and then Replay.
+func RecordAndReplay(cmd []string, wd string, quiet bool) (p *Process, tracedir string, err error) {
+	tracedir, err = Record(cmd, wd, quiet)
+	if tracedir == "" {
+		return nil, "", err
+	}
+	p, err = Replay(tracedir, quiet)
+	return p, tracedir, err
+}

pkg/proc/gdbserial/rr_test.go

+package gdbserial_test
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/derekparker/delve/pkg/proc"
+	"github.com/derekparker/delve/pkg/proc/gdbserial"
+	protest "github.com/derekparker/delve/pkg/proc/test"
+)
+
+func withTestRecording(name string, t testing.TB, fn func(p *gdbserial.Process, fixture protest.Fixture)) {
+	fixture := protest.BuildFixture(name)
+	protest.MustHaveRecordingAllowed(t)
+	if path, _ := exec.LookPath("rr"); path == "" {
+		t.Skip("test skipped, rr not found")
+	}
+	t.Log("recording")
+	p, tracedir, err := gdbserial.RecordAndReplay([]string{fixture.Path}, ".", true)
+	if err != nil {
+		t.Fatal("Launch():", err)
+	}
+	t.Logf("replaying %q", tracedir)
+
+	defer func() {
+		p.Halt()
+		p.Detach(true)
+		if tracedir != "" {
+			protest.SafeRemoveAll(tracedir)
+		}
+	}()
+
+	fn(p, fixture)
+}
+
+func assertNoError(err error, t testing.TB, s string) {
+	if err != nil {
+		_, file, line, _ := runtime.Caller(1)
+		fname := filepath.Base(file)
+		t.Fatalf("failed assertion at %s:%d: %s - %s\n", fname, line, s, err)
+	}
+}
+
+func setFunctionBreakpoint(p proc.Process, t *testing.T, fname string) *proc.Breakpoint {
+	addr, err := proc.FindFunctionLocation(p, fname, true, 0)
+	assertNoError(err, t, fmt.Sprintf("FindFunctionLocation(%s)", fname))
+	bp, err := p.SetBreakpoint(addr, proc.UserBreakpoint, nil)
+	assertNoError(err, t, fmt.Sprintf("SetBreakpoint(%#x) function %s", addr, fname))
+	return bp
+}
+
+func TestRestartAfterExit(t *testing.T) {
+	protest.AllowRecording(t)
+	withTestRecording("testnextprog", t, func(p *gdbserial.Process, fixture protest.Fixture) {
+		setFunctionBreakpoint(p, t, "main.main")
+		assertNoError(proc.Continue(p), t, "Continue")
+		loc, err := p.CurrentThread().Location()
+		assertNoError(err, t, "CurrentThread().Location()")
+		err = proc.Continue(p)
+		if _, isexited := err.(proc.ProcessExitedError); err == nil || !isexited {
+			t.Fatalf("program did not exit: %v", err)
+		}
+
+		assertNoError(p.Restart(""), t, "Restart")
+
+		assertNoError(proc.Continue(p), t, "Continue (after restart)")
+		loc2, err := p.CurrentThread().Location()
+		assertNoError(err, t, "CurrentThread().Location() (after restart)")
+		if loc2.Line != loc.Line {
+			t.Fatalf("stopped at %d (expected %d)", loc2.Line, loc.Line)
+		}
+		err = proc.Continue(p)
+		if _, isexited := err.(proc.ProcessExitedError); err == nil || !isexited {
+			t.Fatalf("program did not exit (after exit): %v", err)
+		}
+	})
+}
+
+func TestRestartDuringStop(t *testing.T) {
+	protest.AllowRecording(t)
+	withTestRecording("testnextprog", t, func(p *gdbserial.Process, fixture protest.Fixture) {
+		setFunctionBreakpoint(p, t, "main.main")
+		assertNoError(proc.Continue(p), t, "Continue")
+		loc, err := p.CurrentThread().Location()
+		assertNoError(err, t, "CurrentThread().Location()")
+
+		assertNoError(p.Restart(""), t, "Restart")
+
+		assertNoError(proc.Continue(p), t, "Continue (after restart)")
+		loc2, err := p.CurrentThread().Location()
+		assertNoError(err, t, "CurrentThread().Location() (after restart)")
+		if loc2.Line != loc.Line {
+			t.Fatalf("stopped at %d (expected %d)", loc2.Line, loc.Line)
+		}
+		err = proc.Continue(p)
+		if _, isexited := err.(proc.ProcessExitedError); err == nil || !isexited {
+			t.Fatalf("program did not exit (after exit): %v", err)
+		}
+	})
+}
+
+func setFileBreakpoint(p proc.Process, t *testing.T, file string, line int) *proc.Breakpoint {
+	addr, _, err := p.BinInfo().LineToPC(file, line)
+	assertNoError(err, t, "LineToPC")
+	bp, err := p.SetBreakpoint(addr, proc.UserBreakpoint, nil)
+	assertNoError(err, t, fmt.Sprintf("SetBreakpoint(%#x) - %s:%d", addr, file, line))
+	return bp
+}
+
+func TestReverseBreakpointCounts(t *testing.T) {
+	protest.AllowRecording(t)
+	withTestRecording("bpcountstest", t, func(p *gdbserial.Process, fixture protest.Fixture) {
+		endbp := setFileBreakpoint(p, t, fixture.Source, 28)
+		assertNoError(proc.Continue(p), t, "Continue()")
+		loc, _ := p.CurrentThread().Location()
+		if loc.PC != endbp.Addr {
+			t.Fatalf("did not reach end of main.main function: %s:%d (%#x)", loc.File, loc.Line, loc.PC)
+		}
+
+		p.ClearBreakpoint(endbp.Addr)
+		assertNoError(p.Direction(proc.Backward), t, "Switching to backward direction")
+		bp := setFileBreakpoint(p, t, fixture.Source, 12)
+		startbp := setFileBreakpoint(p, t, fixture.Source, 20)
+
+	countLoop:
+		for {
+			assertNoError(proc.Continue(p), t, "Continue()")
+			loc, _ := p.CurrentThread().Location()
+			switch loc.PC {
+			case startbp.Addr:
+				break countLoop
+			case bp.Addr:
+				// ok
+			default:
+				t.Fatalf("unexpected stop location %s:%d %#x", loc.File, loc.Line, loc.PC)
+			}
+		}
+
+		t.Logf("TotalHitCount: %d", bp.TotalHitCount)
+		if bp.TotalHitCount != 200 {
+			t.Fatalf("Wrong TotalHitCount for the breakpoint (%d)", bp.TotalHitCount)
+		}
+
+		if len(bp.HitCount) != 2 {
+			t.Fatalf("Wrong number of goroutines for breakpoint (%d)", len(bp.HitCount))
+		}
+
+		for _, v := range bp.HitCount {
+			if v != 100 {
+				t.Fatalf("Wrong HitCount for breakpoint (%v)", bp.HitCount)
+			}
+		}
+	})
+}
+
+func getPosition(p *gdbserial.Process, t *testing.T) (when string, loc *proc.Location) {
+	var err error
+	when, err = p.When()
+	assertNoError(err, t, "When")
+	loc, err = p.CurrentThread().Location()
+	assertNoError(err, t, "Location")
+	return
+}
+
+func TestCheckpoints(t *testing.T) {
+	protest.AllowRecording(t)
+	withTestRecording("continuetestprog", t, func(p *gdbserial.Process, fixture protest.Fixture) {
+		// Continues until start of main.main, record output of 'when'
+		bp := setFunctionBreakpoint(p, t, "main.main")
+		assertNoError(proc.Continue(p), t, "Continue")
+		when0, loc0 := getPosition(p, t)
+		t.Logf("when0: %q (%#x)", when0, loc0.PC)
+
+		// Create a checkpoint and check that the list of checkpoints reflects this
+		cpid, err := p.Checkpoint("checkpoint1")
+		if cpid != 1 {
+			t.Errorf("unexpected checkpoint id %d", cpid)
+		}
+		assertNoError(err, t, "Checkpoint")
+		checkpoints, err := p.Checkpoints()
+		assertNoError(err, t, "Checkpoints")
+		if len(checkpoints) != 1 {
+			t.Fatalf("wrong number of checkpoints %v (one expected)", checkpoints)
+		}
+
+		// Move forward with next, check that the output of 'when' changes
+		assertNoError(proc.Next(p), t, "First Next")
+		assertNoError(proc.Next(p), t, "Second Next")
+		when1, loc1 := getPosition(p, t)
+		t.Logf("when1: %q (%#x)", when1, loc1.PC)
+		if loc0.PC == loc1.PC {
+			t.Fatalf("next did not move process %#x", loc0.PC)
+		}
+		if when0 == when1 {
+			t.Fatalf("output of when did not change after next: %q", when0)
+		}
+
+		// Move back to checkpoint, check that the output of 'when' is the same as
+		// what it was when we set the breakpoint
+		p.Restart(fmt.Sprintf("c%d", cpid))
+		when2, loc2 := getPosition(p, t)
+		t.Logf("when2: %q (%#x)", when2, loc2.PC)
+		if loc2.PC != loc0.PC {
+			t.Fatalf("PC address mismatch %#x != %#x", loc0.PC, loc2.PC)
+		}
+		if when0 != when2 {
+			t.Fatalf("output of when mismatched %q != %q", when0, when2)
+		}
+
+		// Move forward with next again, check that the output of 'when' matches
+		assertNoError(proc.Next(p), t, "First Next")
+		assertNoError(proc.Next(p), t, "Second Next")
+		when3, loc3 := getPosition(p, t)
+		t.Logf("when3: %q (%#x)", when3, loc3.PC)
+		if loc3.PC != loc1.PC {
+			t.Fatalf("PC address mismatch %#x != %#x", loc1.PC, loc3.PC)
+		}
+		if when3 != when1 {
+			t.Fatalf("when output mismatch %q != %q", when1, when3)
+		}
+
+		// Delete breakpoint, move back to checkpoint then next twice and check
+		// output of 'when' again
+		_, err = p.ClearBreakpoint(bp.Addr)
+		assertNoError(err, t, "ClearBreakpoint")
+		p.Restart(fmt.Sprintf("c%d", cpid))
+		assertNoError(proc.Next(p), t, "First Next")
+		assertNoError(proc.Next(p), t, "Second Next")
+		when4, loc4 := getPosition(p, t)
+		t.Logf("when4: %q (%#x)", when4, loc4.PC)
+		if loc4.PC != loc1.PC {
+			t.Fatalf("PC address mismatch %#x != %#x", loc1.PC, loc4.PC)
+		}
+		if when4 != when1 {
+			t.Fatalf("when output mismatch %q != %q", when1, when4)
+		}
+
+		// Delete checkpoint, check that the list of checkpoints is updated
+		assertNoError(p.ClearCheckpoint(cpid), t, "ClearCheckpoint")
+		checkpoints, err = p.Checkpoints()
+		assertNoError(err, t, "Checkpoints")
+		if len(checkpoints) != 0 {
+			t.Fatalf("wrong number of checkpoints %v (zero expected)", checkpoints)
+		}
+	})
+}

pkg/proc/interface.go

@@ -10,6 +10,43 @@ type Process interface {
 	Info
 	ProcessManipulation
 	BreakpointManipulation
+	RecordingManipulation
+}
+
+// RecordingManipulation is an interface for manipulating process recordings.
+type RecordingManipulation interface {
+	// Recorded returns true if the current process is a recording and the path
+	// to the trace directory.
+	Recorded() (recorded bool, tracedir string)
+	// Restart restarts the recording from the specified position, or from the
+	// last checkpoint if pos == "".
+	// If pos starts with 'c' it's a checkpoint ID, otherwise it's an event
+	// number.
+	Restart(pos string) error
+	// Direction changes execution direction.
+	Direction(Direction) error
+	// When returns current recording position.
+	When() (string, error)
+	// Checkpoint sets a checkpoint at the current position.
+	Checkpoint(where string) (id int, err error)
+	// Checkpoints returns the list of currently set checkpoint.
+	Checkpoints() ([]Checkpoint, error)
+	// ClearCheckpoint removes a checkpoint.
+	ClearCheckpoint(id int) error
+}
+
+type Direction int8
+
+const (
+	Forward  Direction = 0
+	Backward Direction = 1
+)
+
+// Checkpoint is a checkpoint
+type Checkpoint struct {
+	ID    int
+	When  string
+	Where string
 }
 
 // Info is an interface that provides general information on the target.

pkg/proc/native/proc.go

@@ -67,6 +67,14 @@ func (dbp *Process) BinInfo() *proc.BinaryInfo {
 	return &dbp.bi
 }
 
+func (dbp *Process) Recorded() (bool, string)                { return false, "" }
+func (dbp *Process) Restart(string) error                    { return proc.NotRecordedErr }
+func (dbp *Process) Direction(proc.Direction) error          { return proc.NotRecordedErr }
+func (dbp *Process) When() (string, error)                   { return "", nil }
+func (dbp *Process) Checkpoint(string) (int, error)          { return -1, proc.NotRecordedErr }
+func (dbp *Process) Checkpoints() ([]proc.Checkpoint, error) { return nil, proc.NotRecordedErr }
+func (dbp *Process) ClearCheckpoint(int) error               { return proc.NotRecordedErr }
+
 // Detach from the process being debugged, optionally killing it.
 func (dbp *Process) Detach(kill bool) (err error) {
 	if dbp.exited {

pkg/proc/proc.go

@@ -20,6 +20,7 @@ type functionDebugInfo struct {
 }
 
 var NotExecutableErr = errors.New("not an executable file")
+var NotRecordedErr = errors.New("not a recording")
 
 // ProcessExitedError indicates that the process has exited and contains both
 // process id and exit status.
@@ -114,7 +115,7 @@ func Continue(dbp Process) error {
 		switch {
 		case curbp == nil:
 			// runtime.Breakpoint or manual stop
-			if onRuntimeBreakpoint(curthread) {
+			if recorded, _ := dbp.Recorded(); onRuntimeBreakpoint(curthread) && !recorded {
 				// Single-step current thread until we exit runtime.breakpoint and
 				// runtime.Breakpoint.
 				// On go < 1.8 it was sufficient to single-step twice on go1.8 a change

pkg/proc/proc_unix_test.go

@@ -17,6 +17,9 @@ func TestIssue419(t *testing.T) {
 		// debugserver bug?
 		return
 	}
+	if testBackend == "rr" {
+		return
+	}
 	// SIGINT directed at the inferior should be passed along not swallowed by delve
 	withTestProcess("issue419", t, func(p proc.Process, fixture protest.Fixture) {
 		_, err := setFunctionBreakpoint(p, "main.main")

pkg/proc/test/support.go

@@ -8,6 +8,8 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
+	"sync"
 	"testing"
 )
 
@@ -92,3 +94,92 @@ func RunTestsWithFixtures(m *testing.M) int {
 	}
 	return status
 }
+
+var recordingAllowed = map[string]bool{}
+var recordingAllowedMu sync.Mutex
+
+// testName returns the name of the test being run using runtime.Caller.
+// On go1.8 t.Name() could be called instead, this is a workaround to
+// support <=go1.7
+func testName(t testing.TB) string {
+	for i := 1; i < 10; i++ {
+		pc, _, _, ok := runtime.Caller(i)
+		if !ok {
+			break
+		}
+		fn := runtime.FuncForPC(pc)
+		if fn == nil {
+			continue
+		}
+		name := fn.Name()
+		v := strings.Split(name, ".")
+		if strings.HasPrefix(v[len(v)-1], "Test") {
+			return name
+		}
+	}
+	return "unknown"
+}
+
+// AllowRecording allows the calling test to be used with a recording of the
+// fixture.
+func AllowRecording(t testing.TB) {
+	recordingAllowedMu.Lock()
+	defer recordingAllowedMu.Unlock()
+	name := testName(t)
+	t.Logf("enabling recording for %s", name)
+	recordingAllowed[name] = true
+}
+
+// MustHaveRecordingAllowed skips this test if recording is not allowed
+//
+// Not all the tests can be run with a recording:
+// - some fixtures never terminate independently (loopprog,
+//   testnextnethttp) and can not be recorded
+// - some tests assume they can interact with the target process (for
+//   example TestIssue419, or anything changing the value of a variable),
+//   which we can't do on with a recording
+// - some tests assume that the Pid returned by the process is valid, but
+//   it won't be at replay time
+// - some tests will start the fixture but not never execute a single
+//   instruction, for some reason rr doesn't like this and will print an
+//   error if it happens
+// - many tests will assume that we can return from a runtime.Breakpoint,
+//   with a recording this is not possible because when the fixture ran it
+//   wasn't attached to a debugger and in those circumstances a
+//   runtime.Breakpoint leads directly to a crash
+//
+// Some of the tests using runtime.Breakpoint (anything involving variable
+// evaluation and TestWorkDir) have been adapted to work with a recording.
+func MustHaveRecordingAllowed(t testing.TB) {
+	recordingAllowedMu.Lock()
+	defer recordingAllowedMu.Unlock()
+	name := testName(t)
+	if !recordingAllowed[name] {
+		t.Skipf("recording not allowed for %s", name)
+	}
+}
+
+// SafeRemoveAll removes dir and its contents but only as long as dir does
+// not contain directories.
+func SafeRemoveAll(dir string) {
+	dh, err := os.Open(dir)
+	if err != nil {
+		return
+	}
+	defer dh.Close()
+	fis, err := dh.Readdir(-1)
+	if err != nil {
+		return
+	}
+	for _, fi := range fis {
+		if fi.IsDir() {
+			return
+		}
+	}
+	for _, fi := range fis {
+		if err := os.Remove(filepath.Join(dir, fi.Name())); err != nil {
+			return
+		}
+	}
+	os.Remove(dir)
+}

pkg/terminal/command.go

@@ -218,6 +218,41 @@ Supported commands: print, stack and goroutine)`},
 Specifies that the breakpoint or tracepoint should break only if the boolean expression is true.`},
 	}
 
+	if client == nil || client.Recorded() {
+		c.cmds = append(c.cmds, command{
+			aliases: []string{"rewind", "rw"},
+			cmdFn:   rewind,
+			helpMsg: "Run backwards until breakpoint or program termination.",
+		})
+		c.cmds = append(c.cmds, command{
+			aliases: []string{"check", "checkpoint"},
+			cmdFn:   checkpoint,
+			helpMsg: `Creates a checkpoint at the current position.
+			
+	checkpoint [where]`,
+		})
+		c.cmds = append(c.cmds, command{
+			aliases: []string{"checkpoints"},
+			cmdFn:   checkpoints,
+			helpMsg: "Print out info for existing checkpoints.",
+		})
+		c.cmds = append(c.cmds, command{
+			aliases: []string{"clear-checkpoint", "clearcheck"},
+			cmdFn:   clearCheckpoint,
+			helpMsg: `Deletes checkpoint.
+			
+	checkpoint <id>`,
+		})
+		for i := range c.cmds {
+			v := &c.cmds[i]
+			if v.match("restart") {
+				v.helpMsg = `Restart process from a checkpoint or event.
+	
+	restart [event number or checkpoint id]`
+			}
+		}
+	}
+
 	sort.Sort(ByFirstAlias(c.cmds))
 	return c
 }
@@ -569,14 +604,24 @@ func writeGoroutineLong(w io.Writer, g *api.Goroutine, prefix string) {
 }
 
 func restart(t *Term, ctx callContext, args string) error {
-	discarded, err := t.client.Restart()
+	discarded, err := t.client.RestartFrom(args)
 	if err != nil {
 		return err
 	}
-	fmt.Println("Process restarted with PID", t.client.ProcessPid())
+	if !t.client.Recorded() {
+		fmt.Println("Process restarted with PID", t.client.ProcessPid())
+	}
 	for i := range discarded {
 		fmt.Printf("Discarded %s at %s: %v\n", formatBreakpointName(discarded[i].Breakpoint, false), formatBreakpointLocation(discarded[i].Breakpoint), discarded[i].Reason)
 	}
+	if t.client.Recorded() {
+		state, err := t.client.GetState()
+		if err != nil {
+			return err
+		}
+		printcontext(t, state)
+		printfile(t, state.CurrentThread.File, state.CurrentThread.Line, true)
+	}
 	return nil
 }
 
@@ -1196,6 +1241,10 @@ func printcontext(t *Term, state *api.DebuggerState) error {
 
 	printcontextThread(t, state.CurrentThread)
 
+	if state.When != "" {
+		fmt.Println(state.When)
+	}
+
 	return nil
 }
 
@@ -1409,6 +1458,74 @@ func (c *Commands) executeFile(t *Term, name string) error {
 	return scanner.Err()
 }
 
+func rewind(t *Term, ctx callContext, args string) error {
+	stateChan := t.client.Rewind()
+	var state *api.DebuggerState
+	for state = range stateChan {
+		if state.Err != nil {
+			return state.Err
+		}
+		printcontext(t, state)
+	}
+	printfile(t, state.CurrentThread.File, state.CurrentThread.Line, true)
+	return nil
+}
+
+func checkpoint(t *Term, ctx callContext, args string) error {
+	if args == "" {
+		state, err := t.client.GetState()
+		if err != nil {
+			return err
+		}
+		var loc api.Location = api.Location{PC: state.CurrentThread.PC, File: state.CurrentThread.File, Line: state.CurrentThread.Line, Function: state.CurrentThread.Function}
+		if state.SelectedGoroutine != nil {
+			loc = state.SelectedGoroutine.CurrentLoc
+		}
+		fname := "???"
+		if loc.Function != nil {
+			fname = loc.Function.Name
+		}
+		args = fmt.Sprintf("%s() %s:%d (%#x)", fname, loc.File, loc.Line, loc.PC)
+	}
+
+	cpid, err := t.client.Checkpoint(args)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Checkpoint c%d created.\n", cpid)
+	return nil
+}
+
+func checkpoints(t *Term, ctx callContext, args string) error {
+	cps, err := t.client.ListCheckpoints()
+	if err != nil {
+		return err
+	}
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, 4, 4, 2, ' ', 0)
+	fmt.Fprintln(w, "ID\tWhen\tWhere")
+	for _, cp := range cps {
+		fmt.Fprintf(w, "c%d\t%s\t%s\n", cp.ID, cp.When, cp.Where)
+	}
+	w.Flush()
+	return nil
+}
+
+func clearCheckpoint(t *Term, ctx callContext, args string) error {
+	if len(args) < 0 {
+		return errors.New("not enough arguments to clear-checkpoint")
+	}
+	if args[0] != 'c' {
+		return errors.New("clear-checkpoint argument must be a checkpoint ID")
+	}
+	id, err := strconv.Atoi(args[1:])
+	if err != nil {
+		return errors.New("clear-checkpoint argument must be a checkpoint ID")
+	}
+	return t.client.ClearCheckpoint(id)
+}
+
 func formatBreakpointName(bp *api.Breakpoint, upcase bool) string {
 	thing := "breakpoint"
 	if bp.Tracepoint {

pkg/terminal/command_test.go

@@ -86,6 +86,9 @@ func (ft *FakeTerminal) AssertExecError(cmdstr, tgterr string) {
 }
 
 func withTestTerminal(name string, t testing.TB, fn func(*FakeTerminal)) {
+	if testBackend == "rr" {
+		test.MustHaveRecordingAllowed(t)
+	}
 	os.Setenv("TERM", "dumb")
 	listener, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -103,6 +106,9 @@ func withTestTerminal(name string, t testing.TB, fn func(*FakeTerminal)) {
 	client := rpc2.NewClient(listener.Addr().String())
 	defer func() {
 		client.Detach(true)
+		if dir, _ := client.TraceDirectory(); dir != "" {
+			test.SafeRemoveAll(dir)
+		}
 	}()
 
 	ft := &FakeTerminal{
@@ -207,6 +213,7 @@ func TestIssue354(t *testing.T) {
 }
 
 func TestIssue411(t *testing.T) {
+	test.AllowRecording(t)
 	withTestTerminal("math", t, func(term *FakeTerminal) {
 		term.MustExec("break math.go:8")
 		term.MustExec("trace math.go:9")
@@ -221,6 +228,7 @@ func TestIssue411(t *testing.T) {
 func TestScopePrefix(t *testing.T) {
 	const goroutinesLinePrefix = "  Goroutine "
 	const goroutinesCurLinePrefix = "* Goroutine "
+	test.AllowRecording(t)
 	withTestTerminal("goroutinestackprog", t, func(term *FakeTerminal) {
 		term.MustExec("b stacktraceme")
 		term.MustExec("continue")
@@ -344,6 +352,7 @@ func TestScopePrefix(t *testing.T) {
 
 func TestOnPrefix(t *testing.T) {
 	const prefix = "\ti: "
+	test.AllowRecording(t)
 	withTestTerminal("goroutinestackprog", t, func(term *FakeTerminal) {
 		term.MustExec("b agobp main.agoroutine")
 		term.MustExec("on agobp print i")
@@ -384,6 +393,7 @@ func TestOnPrefix(t *testing.T) {
 }
 
 func TestNoVars(t *testing.T) {
+	test.AllowRecording(t)
 	withTestTerminal("locationsUpperCase", t, func(term *FakeTerminal) {
 		term.MustExec("b main.main")
 		term.MustExec("continue")
@@ -395,6 +405,7 @@ func TestNoVars(t *testing.T) {
 
 func TestOnPrefixLocals(t *testing.T) {
 	const prefix = "\ti: "
+	test.AllowRecording(t)
 	withTestTerminal("goroutinestackprog", t, func(term *FakeTerminal) {
 		term.MustExec("b agobp main.agoroutine")
 		term.MustExec("on agobp args -v")
@@ -449,6 +460,7 @@ func countOccourences(s string, needle string) int {
 
 func TestIssue387(t *testing.T) {
 	// a breakpoint triggering during a 'next' operation will interrupt it
+	test.AllowRecording(t)
 	withTestTerminal("issue387", t, func(term *FakeTerminal) {
 		breakpointHitCount := 0
 		term.MustExec("break dostuff")
@@ -498,13 +510,13 @@ func listIsAt(t *testing.T, term *FakeTerminal, listcmd string, cur, start, end
 
 	outStart, outEnd := 0, 0
 
-	for i, line := range lines[1:] {
+	for _, line := range lines[1:] {
 		if line == "" {
 			continue
 		}
 		v := re.FindStringSubmatch(line)
 		if len(v) != 3 {
-			t.Fatalf("Could not parse line %d: %q\n", i+1, line)
+			continue
 		}
 		curline, _ := strconv.Atoi(v[2])
 		if v[1] == "=>" {
@@ -518,8 +530,10 @@ func listIsAt(t *testing.T, term *FakeTerminal, listcmd string, cur, start, end
 		outEnd = curline
 	}
 
-	if outStart != start || outEnd != end {
-		t.Fatalf("Wrong output range, got %d:%d expected %d:%d", outStart, outEnd, start, end)
+	if start != -1 || end != -1 {
+		if outStart != start || outEnd != end {
+			t.Fatalf("Wrong output range, got %d:%d expected %d:%d", outStart, outEnd, start, end)
+		}
 	}
 }
 
@@ -537,3 +551,33 @@ func TestListCmd(t *testing.T) {
 		}
 	})
 }
+
+func TestReverseContinue(t *testing.T) {
+	test.AllowRecording(t)
+	if testBackend != "rr" {
+		return
+	}
+	withTestTerminal("continuetestprog", t, func(term *FakeTerminal) {
+		term.MustExec("break main.main")
+		term.MustExec("break main.sayhi")
+		listIsAt(t, term, "continue", 16, -1, -1)
+		listIsAt(t, term, "continue", 12, -1, -1)
+		listIsAt(t, term, "rewind", 16, -1, -1)
+	})
+}
+
+func TestCheckpoints(t *testing.T) {
+	test.AllowRecording(t)
+	if testBackend != "rr" {
+		return
+	}
+	withTestTerminal("continuetestprog", t, func(term *FakeTerminal) {
+		term.MustExec("break main.main")
+		listIsAt(t, term, "continue", 16, -1, -1)
+		term.MustExec("checkpoint")
+		term.MustExec("checkpoints")
+		listIsAt(t, term, "next", 17, -1, -1)
+		listIsAt(t, term, "next", 18, -1, -1)
+		listIsAt(t, term, "restart c1", 16, -1, -1)
+	})
+}

service/api/conversions.go

@@ -274,3 +274,7 @@ func ConvertRegisters(in []proc.Register) (out []Register) {
 	}
 	return
 }
+
+func ConvertCheckpoint(in proc.Checkpoint) (out Checkpoint) {
+	return Checkpoint{ID: in.ID, When: in.When, Where: in.Where}
+}

service/api/types.go

@@ -29,6 +29,8 @@ type DebuggerState struct {
 	// Exited indicates whether the debugged process has exited.
 	Exited     bool `json:"exited"`
 	ExitStatus int  `json:"exitStatus"`
+	// When contains a description of the current position in a recording
+	When string
 	// Filled by RPCClient.Continue, indicates an error
 	Err error `json:"-"`
 }
@@ -236,6 +238,8 @@ type EvalScope struct {
 const (
 	// Continue resumes process execution.
 	Continue = "continue"
+	// Rewind resumes process execution backwards (target must be a recording).
+	Rewind = "rewind"
 	// Step continues to next source line, entering function calls.
 	Step = "step"
 	// StepOut continues to the return address of the current function
@@ -318,3 +322,9 @@ type DiscardedBreakpoint struct {
 	Breakpoint *Breakpoint
 	Reason     string
 }
+
+type Checkpoint struct {
+	ID    int
+	When  string
+	Where string
+}

service/client.go

@@ -20,12 +20,16 @@ type Client interface {
 
 	// Restarts program.
 	Restart() ([]api.DiscardedBreakpoint, error)
+	// Restarts program from the specified position.
+	RestartFrom(pos string) ([]api.DiscardedBreakpoint, error)
 
 	// GetState returns the current debugger state.
 	GetState() (*api.DebuggerState, error)
 
 	// Continue resumes process execution.
 	Continue() <-chan *api.DebuggerState
+	// Rewind resumes process execution backwards.
+	Rewind() <-chan *api.DebuggerState
 	// Next continues to the next source line, not entering function calls.
 	Next() (*api.DebuggerState, error)
 	// Step continues to the next source line, entering function calls.
@@ -112,4 +116,15 @@ type Client interface {
 	DisassembleRange(scope api.EvalScope, startPC, endPC uint64, flavour api.AssemblyFlavour) (api.AsmInstructions, error)
 	// Disassemble code of the function containing PC
 	DisassemblePC(scope api.EvalScope, pc uint64, flavour api.AssemblyFlavour) (api.AsmInstructions, error)
+
+	// Recorded returns true if the target is a recording.
+	Recorded() bool
+	// TraceDirectory returns the path to the trace directory for a recording.
+	TraceDirectory() (string, error)
+	// Checkpoint sets a checkpoint at the current position.
+	Checkpoint(where string) (checkpointID int, err error)
+	// ListCheckpoints gets all checkpoints.
+	ListCheckpoints() ([]api.Checkpoint, error)
+	// ClearCheckpoint removes a checkpoint
+	ClearCheckpoint(id int) error
 }

service/debugger/debugger.go

@@ -78,8 +78,16 @@ func New(config *Config) (*Debugger, error) {
 		d.target = p
 
 	case d.config.CoreFile != "":
-		log.Printf("opening core file %s (executable %s)", d.config.CoreFile, d.config.ProcessArgs[0])
-		p, err := core.OpenCore(d.config.CoreFile, d.config.ProcessArgs[0])
+		var p proc.Process
+		var err error
+		switch d.config.Backend {
+		case "rr":
+			log.Printf("opening trace %s", d.config.CoreFile)
+			p, err = gdbserial.Replay(d.config.CoreFile, false)
+		default:
+			log.Printf("opening core file %s (executable %s)", d.config.CoreFile, d.config.ProcessArgs[0])
+			p, err = core.OpenCore(d.config.CoreFile, d.config.ProcessArgs[0])
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -105,6 +113,9 @@ func (d *Debugger) Launch(processArgs []string, wd string) (proc.Process, error)
 		return native.Launch(processArgs, wd)
 	case "lldb":
 		return gdbserial.LLDBLaunch(processArgs, wd)
+	case "rr":
+		p, _, err := gdbserial.RecordAndReplay(processArgs, wd, false)
+		return p, err
 	case "default":
 		if runtime.GOOS == "darwin" {
 			return gdbserial.LLDBLaunch(processArgs, wd)
@@ -173,12 +184,19 @@ func (d *Debugger) detach(kill bool) error {
 
 // Restart will restart the target process, first killing
 // and then exec'ing it again.
-func (d *Debugger) Restart() ([]api.DiscardedBreakpoint, error) {
+// If the target process is a recording it will restart it from the given
+// position. If pos starts with 'c' it's a checkpoint ID, otherwise it's an
+// event number.
+func (d *Debugger) Restart(pos string) ([]api.DiscardedBreakpoint, error) {
 	d.processMutex.Lock()
 	defer d.processMutex.Unlock()
 
-	if d.config.CoreFile != "" {
-		return nil, errors.New("can not restart core dump")
+	if recorded, _ := d.target.Recorded(); recorded {
+		return nil, d.target.Restart(pos)
+	}
+
+	if pos != "" {
+		return nil, proc.NotRecordedErr
 	}
 
 	if !d.target.Exited() {
@@ -203,6 +221,7 @@ func (d *Debugger) Restart() ([]api.DiscardedBreakpoint, error) {
 			continue
 		}
 		if len(oldBp.File) > 0 {
+			var err error
 			oldBp.Addr, err = proc.FindFileLocation(p, oldBp.File, oldBp.Line)
 			if err != nil {
 				discarded = append(discarded, api.DiscardedBreakpoint{oldBp, err.Error()})
@@ -262,6 +281,10 @@ func (d *Debugger) state() (*api.DebuggerState, error) {
 		}
 	}
 
+	if recorded, _ := d.target.Recorded(); recorded {
+		state.When, _ = d.target.When()
+	}
+
 	return state, nil
 }
 
@@ -498,6 +521,32 @@ func (d *Debugger) Command(command *api.DebuggerCommand) (*api.DebuggerState, er
 		err = d.collectBreakpointInformation(state)
 		return state, err
 
+	case api.Rewind:
+		log.Print("rewinding")
+		if err := d.target.Direction(proc.Backward); err != nil {
+			return nil, err
+		}
+		defer func() {
+			d.target.Direction(proc.Forward)
+		}()
+		err = proc.Continue(d.target)
+		if err != nil {
+			if exitedErr, exited := err.(proc.ProcessExitedError); exited {
+				state := &api.DebuggerState{}
+				state.Exited = true
+				state.ExitStatus = exitedErr.Status
+				state.Err = errors.New(exitedErr.Error())
+				return state, nil
+			}
+			return nil, err
+		}
+		state, stateErr := d.state()
+		if stateErr != nil {
+			return state, stateErr
+		}
+		err = d.collectBreakpointInformation(state)
+		return state, err
+
 	case api.Next:
 		log.Print("nexting")
 		err = proc.Next(d.target)
@@ -897,3 +946,36 @@ func (d *Debugger) Disassemble(scope api.EvalScope, startPC, endPC uint64, flavo
 
 	return disass, nil
 }
+
+// Recorded returns true if the target is a recording.
+func (d *Debugger) Recorded() (recorded bool, tracedir string) {
+	d.processMutex.Lock()
+	defer d.processMutex.Unlock()
+	return d.target.Recorded()
+}
+
+func (d *Debugger) Checkpoint(where string) (int, error) {
+	d.processMutex.Lock()
+	defer d.processMutex.Unlock()
+	return d.target.Checkpoint(where)
+}
+
+func (d *Debugger) Checkpoints() ([]api.Checkpoint, error) {
+	d.processMutex.Lock()
+	defer d.processMutex.Unlock()
+	cps, err := d.target.Checkpoints()
+	if err != nil {
+		return nil, err
+	}
+	r := make([]api.Checkpoint, len(cps))
+	for i := range cps {
+		r[i] = api.ConvertCheckpoint(cps[i])
+	}
+	return r, nil
+}
+
+func (d *Debugger) ClearCheckpoint(id int) error {
+	d.processMutex.Lock()
+	defer d.processMutex.Unlock()
+	return d.target.ClearCheckpoint(id)
+}

service/rpc1/server.go

@@ -36,7 +36,7 @@ func (s *RPCServer) Restart(arg1 interface{}, arg2 *int) error {
 	if s.config.AttachPid != 0 {
 		return errors.New("cannot restart process Delve did not create")
 	}
-	_, err := s.debugger.Restart()
+	_, err := s.debugger.Restart("")
 	return err
 }
 

service/rpc2/client.go

@@ -51,7 +51,13 @@ func (c *RPCClient) Detach(kill bool) error {
 
 func (c *RPCClient) Restart() ([]api.DiscardedBreakpoint, error) {
 	out := new(RestartOut)
-	err := c.call("Restart", RestartIn{}, out)
+	err := c.call("Restart", RestartIn{""}, out)
+	return out.DiscardedBreakpoints, err
+}
+
+func (c *RPCClient) RestartFrom(pos string) ([]api.DiscardedBreakpoint, error) {
+	out := new(RestartOut)
+	err := c.call("Restart", RestartIn{pos}, out)
 	return out.DiscardedBreakpoints, err
 }
 
@@ -62,11 +68,19 @@ func (c *RPCClient) GetState() (*api.DebuggerState, error) {
 }
 
 func (c *RPCClient) Continue() <-chan *api.DebuggerState {
+	return c.continueDir(api.Continue)
+}
+
+func (c *RPCClient) Rewind() <-chan *api.DebuggerState {
+	return c.continueDir(api.Rewind)
+}
+
+func (c *RPCClient) continueDir(cmd string) <-chan *api.DebuggerState {
 	ch := make(chan *api.DebuggerState)
 	go func() {
 		for {
 			out := new(CommandOut)
-			err := c.call("Command", &api.DebuggerCommand{Name: api.Continue}, &out)
+			err := c.call("Command", &api.DebuggerCommand{Name: cmd}, &out)
 			state := out.State
 			if err != nil {
 				state.Err = err
@@ -299,6 +313,41 @@ func (c *RPCClient) DisassemblePC(scope api.EvalScope, pc uint64, flavour api.As
 	return out.Disassemble, err
 }
 
+// Recorded returns true if the debugger target is a recording.
+func (c *RPCClient) Recorded() bool {
+	out := new(RecordedOut)
+	c.call("Recorded", RecordedIn{}, out)
+	return out.Recorded
+}
+
+// TraceDirectory returns the path to the trace directory for a recording.
+func (c *RPCClient) TraceDirectory() (string, error) {
+	var out RecordedOut
+	err := c.call("Recorded", RecordedIn{}, &out)
+	return out.TraceDirectory, err
+}
+
+// Checkpoint sets a checkpoint at the current position.
+func (c *RPCClient) Checkpoint(where string) (checkpointID int, err error) {
+	var out CheckpointOut
+	err = c.call("Checkpoint", CheckpointIn{where}, &out)
+	return out.ID, err
+}
+
+// ListCheckpoints gets all checkpoints.
+func (c *RPCClient) ListCheckpoints() ([]api.Checkpoint, error) {
+	var out ListCheckpointsOut
+	err := c.call("ListCheckpoints", ListCheckpointsIn{}, &out)
+	return out.Checkpoints, err
+}
+
+// ClearCheckpoint removes a checkpoint
+func (c *RPCClient) ClearCheckpoint(id int) error {
+	var out ClearCheckpointOut
+	err := c.call("ClearCheckpoint", ClearCheckpointIn{id}, &out)
+	return err
+}
+
 func (c *RPCClient) url(path string) string {
 	return fmt.Sprintf("http://%s%s", c.addr, path)
 }

service/rpc2/server.go

@@ -59,6 +59,9 @@ func (s *RPCServer) Detach(arg DetachIn, out *DetachOut) error {
 }
 
 type RestartIn struct {
+	// Position to restart from, if it starts with 'c' it's a checkpoint ID,
+	// otherwise it's an event number. Only valid for recorded targets.
+	Position string
 }
 
 type RestartOut struct {
@@ -71,7 +74,7 @@ func (s *RPCServer) Restart(arg RestartIn, out *RestartOut) error {
 		return errors.New("cannot restart process Delve did not create")
 	}
 	var err error
-	out.DiscardedBreakpoints, err = s.debugger.Restart()
+	out.DiscardedBreakpoints, err = s.debugger.Restart(arg.Position)
 	return err
 }
 
@@ -573,3 +576,54 @@ func (c *RPCServer) Disassemble(arg DisassembleIn, out *DisassembleOut) error {
 	out.Disassemble, err = c.debugger.Disassemble(arg.Scope, arg.StartPC, arg.EndPC, arg.Flavour)
 	return err
 }
+
+type RecordedIn struct {
+}
+
+type RecordedOut struct {
+	Recorded       bool
+	TraceDirectory string
+}
+
+func (s *RPCServer) Recorded(arg RecordedIn, out *RecordedOut) error {
+	out.Recorded, out.TraceDirectory = s.debugger.Recorded()
+	return nil
+}
+
+type CheckpointIn struct {
+	Where string
+}
+
+type CheckpointOut struct {
+	ID int
+}
+
+func (s *RPCServer) Checkpoint(arg CheckpointIn, out *CheckpointOut) error {
+	var err error
+	out.ID, err = s.debugger.Checkpoint(arg.Where)
+	return err
+}
+
+type ListCheckpointsIn struct {
+}
+
+type ListCheckpointsOut struct {
+	Checkpoints []api.Checkpoint
+}
+
+func (s *RPCServer) ListCheckpoints(arg ListCheckpointsIn, out *ListCheckpointsOut) error {
+	var err error
+	out.Checkpoints, err = s.debugger.Checkpoints()
+	return err
+}
+
+type ClearCheckpointIn struct {
+	ID int
+}
+
+type ClearCheckpointOut struct {
+}
+
+func (s *RPCServer) ClearCheckpoint(arg ClearCheckpointIn, out *ClearCheckpointOut) error {
+	return s.debugger.ClearCheckpoint(arg.ID)
+}

service/test/integration1_test.go

@@ -21,6 +21,9 @@ import (
 )
 
 func withTestClient1(name string, t *testing.T, fn func(c *rpc1.RPCClient)) {
+	if testBackend == "rr" {
+		protest.MustHaveRecordingAllowed(t)
+	}
 	listener, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		t.Fatalf("couldn't start listener: %s\n", err)
@@ -43,6 +46,12 @@ func withTestClient1(name string, t *testing.T, fn func(c *rpc1.RPCClient)) {
 }
 
 func Test1RunWithInvalidPath(t *testing.T) {
+	if testBackend == "rr" {
+		// This test won't work because rr returns an error, after recording, when
+		// the recording failed but also when the recording succeeded but the
+		// inferior returned an error. Therefore we have to ignore errors from rr.
+		return
+	}
 	listener, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		t.Fatalf("couldn't start listener: %s\n", err)

service/test/integration2_test.go

@@ -38,6 +38,9 @@ func TestMain(m *testing.M) {
 }
 
 func withTestClient2(name string, t *testing.T, fn func(c service.Client)) {
+	if testBackend == "rr" {
+		protest.MustHaveRecordingAllowed(t)
+	}
 	listener, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		t.Fatalf("couldn't start listener: %s\n", err)
@@ -54,12 +57,21 @@ func withTestClient2(name string, t *testing.T, fn func(c service.Client)) {
 	client := rpc2.NewClient(listener.Addr().String())
 	defer func() {
 		client.Detach(true)
+		if dir, _ := client.TraceDirectory(); dir != "" {
+			protest.SafeRemoveAll(dir)
+		}
 	}()
 
 	fn(client)
 }
 
 func TestRunWithInvalidPath(t *testing.T) {
+	if testBackend == "rr" {
+		// This test won't work because rr returns an error, after recording, when
+		// the recording failed but also when the recording succeeded but the
+		// inferior returned an error. Therefore we have to ignore errors from rr.
+		return
+	}
 	listener, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		t.Fatalf("couldn't start listener: %s\n", err)
@@ -97,6 +109,7 @@ func TestRestart_afterExit(t *testing.T) {
 }
 
 func TestRestart_breakpointPreservation(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("continuetestprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: 1, Name: "firstbreakpoint", Tracepoint: true})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -167,6 +180,7 @@ func TestRestart_attachPid(t *testing.T) {
 }
 
 func TestClientServer_exit(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("continuetestprog", t, func(c service.Client) {
 		state, err := c.GetState()
 		if err != nil {
@@ -190,6 +204,7 @@ func TestClientServer_exit(t *testing.T) {
 }
 
 func TestClientServer_step(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.helloworld", Line: -1})
 		if err != nil {
@@ -213,6 +228,7 @@ func TestClientServer_step(t *testing.T) {
 }
 
 func TestClientServer_stepout(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.helloworld", Line: -1})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -230,6 +246,7 @@ func TestClientServer_stepout(t *testing.T) {
 }
 
 func testnext2(testcases []nextTest, initialLocation string, t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {
 		bp, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: initialLocation, Line: -1})
 		if err != nil {
@@ -321,6 +338,7 @@ func TestNextFunctionReturn(t *testing.T) {
 }
 
 func TestClientServer_breakpointInMainThread(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testprog", t, func(c service.Client) {
 		bp, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.helloworld", Line: 1})
 		if err != nil {
@@ -342,6 +360,7 @@ func TestClientServer_breakpointInMainThread(t *testing.T) {
 }
 
 func TestClientServer_breakpointInSeparateGoroutine(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testthreads", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.anotherthread", Line: 1})
 		if err != nil {
@@ -396,6 +415,7 @@ func TestClientServer_clearBreakpoint(t *testing.T) {
 }
 
 func TestClientServer_switchThread(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {
 		// With invalid thread id
 		_, err := c.SwitchThread(-1)
@@ -439,6 +459,7 @@ func TestClientServer_switchThread(t *testing.T) {
 }
 
 func TestClientServer_infoLocals(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {
 		fp := testProgPath(t, "testnextprog")
 		_, err := c.CreateBreakpoint(&api.Breakpoint{File: fp, Line: 23})
@@ -460,6 +481,7 @@ func TestClientServer_infoLocals(t *testing.T) {
 }
 
 func TestClientServer_infoArgs(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testnextprog", t, func(c service.Client) {
 		fp := testProgPath(t, "testnextprog")
 		_, err := c.CreateBreakpoint(&api.Breakpoint{File: fp, Line: 47})
@@ -488,6 +510,7 @@ func TestClientServer_infoArgs(t *testing.T) {
 }
 
 func TestClientServer_traceContinue(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("integrationprog", t, func(c service.Client) {
 		fp := testProgPath(t, "integrationprog")
 		_, err := c.CreateBreakpoint(&api.Breakpoint{File: fp, Line: 15, Tracepoint: true, Goroutine: true, Stacktrace: 5, Variables: []string{"i"}})
@@ -545,6 +568,7 @@ func TestClientServer_traceContinue(t *testing.T) {
 }
 
 func TestClientServer_traceContinue2(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("integrationprog", t, func(c service.Client) {
 		bp1, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: 1, Tracepoint: true})
 		if err != nil {
@@ -750,6 +774,7 @@ func TestClientServer_SetVariable(t *testing.T) {
 }
 
 func TestClientServer_FullStacktrace(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("goroutinestackprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.stacktraceme", Line: -1})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -823,6 +848,7 @@ func TestClientServer_FullStacktrace(t *testing.T) {
 
 func TestIssue355(t *testing.T) {
 	// After the target process has terminated should return an error but not crash
+	protest.AllowRecording(t)
 	withTestClient2("continuetestprog", t, func(c service.Client) {
 		bp, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.sayhi", Line: -1})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -854,9 +880,13 @@ func TestIssue355(t *testing.T) {
 		_, err = c.Halt()
 		assertError(err, t, "Halt()")
 		_, err = c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: -1})
-		assertError(err, t, "CreateBreakpoint()")
+		if testBackend != "rr" {
+			assertError(err, t, "CreateBreakpoint()")
+		}
 		_, err = c.ClearBreakpoint(bp.ID)
-		assertError(err, t, "ClearBreakpoint()")
+		if testBackend != "rr" {
+			assertError(err, t, "ClearBreakpoint()")
+		}
 		_, err = c.ListThreads()
 		assertError(err, t, "ListThreads()")
 		_, err = c.GetThread(tid)
@@ -986,6 +1016,7 @@ func TestDisasm(t *testing.T) {
 
 func TestNegativeStackDepthBug(t *testing.T) {
 	// After the target process has terminated should return an error but not crash
+	protest.AllowRecording(t)
 	withTestClient2("continuetestprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.sayhi", Line: -1})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -998,6 +1029,7 @@ func TestNegativeStackDepthBug(t *testing.T) {
 }
 
 func TestClientServer_CondBreakpoint(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("parallel_next", t, func(c service.Client) {
 		bp, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.sayhi", Line: 1})
 		assertNoError(err, t, "CreateBreakpoint()")
@@ -1095,6 +1127,7 @@ func TestIssue419(t *testing.T) {
 }
 
 func TestTypesCommand(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("testvariables2", t, func(c service.Client) {
 		state := <-c.Continue()
 		assertNoError(state.Err, t, "Continue()")
@@ -1121,6 +1154,7 @@ func TestTypesCommand(t *testing.T) {
 }
 
 func TestIssue406(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("issue406", t, func(c service.Client) {
 		locs, err := c.FindLocation(api.EvalScope{-1, 0}, "issue406.go:146")
 		assertNoError(err, t, "FindLocation()")
@@ -1191,6 +1225,7 @@ func TestClientServer_FpRegisters(t *testing.T) {
 		{"XMM7", "0x40026666666666664002666666666666"},
 		{"XMM8", "0x4059999a404ccccd4059999a404ccccd"},
 	}
+	protest.AllowRecording(t)
 	withTestClient2("fputest/", t, func(c service.Client) {
 		<-c.Continue()
 		regs, err := c.ListRegisters(0, true)
@@ -1216,11 +1251,15 @@ func TestClientServer_FpRegisters(t *testing.T) {
 }
 
 func TestClientServer_RestartBreakpointPosition(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestClient2("locationsprog2", t, func(c service.Client) {
 		bpBefore, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.afunction", Line: -1, Tracepoint: true, Name: "this"})
 		addrBefore := bpBefore.Addr
 		t.Logf("%x\n", bpBefore.Addr)
 		assertNoError(err, t, "CreateBreakpoint")
+		stateCh := c.Continue()
+		for range stateCh {
+		}
 		_, err = c.Halt()
 		assertNoError(err, t, "Halt")
 		_, err = c.Restart()
@@ -1242,6 +1281,7 @@ func TestClientServer_SelectedGoroutineLoc(t *testing.T) {
 	// CurrentLocation of SelectedGoroutine should reflect what's happening on
 	// the thread running the goroutine, not the position the goroutine was in
 	// the last time it was parked.
+	protest.AllowRecording(t)
 	withTestClient2("testprog", t, func(c service.Client) {
 		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: -11})
 		assertNoError(err, t, "CreateBreakpoint")
@@ -1260,3 +1300,38 @@ func TestClientServer_SelectedGoroutineLoc(t *testing.T) {
 		}
 	})
 }
+
+func TestClientServer_ReverseContinue(t *testing.T) {
+	protest.AllowRecording(t)
+	if testBackend != "rr" {
+		t.Skip("backend is not rr")
+	}
+	withTestClient2("continuetestprog", t, func(c service.Client) {
+		_, err := c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.main", Line: -1})
+		assertNoError(err, t, "CreateBreakpoint(main.main)")
+		_, err = c.CreateBreakpoint(&api.Breakpoint{FunctionName: "main.sayhi", Line: -1})
+		assertNoError(err, t, "CreateBreakpoint(main.sayhi)")
+
+		state := <-c.Continue()
+		assertNoError(state.Err, t, "first continue")
+		mainPC := state.CurrentThread.PC
+		t.Logf("after first continue %#x", mainPC)
+
+		state = <-c.Continue()
+		assertNoError(state.Err, t, "second continue")
+		sayhiPC := state.CurrentThread.PC
+		t.Logf("after second continue %#x", sayhiPC)
+
+		if mainPC == sayhiPC {
+			t.Fatalf("expected different PC after second PC (%#x)", mainPC)
+		}
+
+		state = <-c.Rewind()
+		assertNoError(state.Err, t, "rewind")
+
+		if mainPC != state.CurrentThread.PC {
+			t.Fatalf("Expected rewind to go back to the first breakpoint: %#x", state.CurrentThread.PC)
+		}
+	})
+
+}

service/test/variables_test.go

@@ -55,11 +55,34 @@ func assertVariable(t *testing.T, variable *proc.Variable, expected varTest) {
 	}
 }
 
-func evalVariable(p proc.Process, symbol string, cfg proc.LoadConfig) (*proc.Variable, error) {
-	scope, err := proc.GoroutineScope(p.CurrentThread())
+func findFirstNonRuntimeFrame(p proc.Process) (proc.Stackframe, error) {
+	frames, err := proc.ThreadStacktrace(p.CurrentThread(), 10)
 	if err != nil {
-		return nil, err
+		return proc.Stackframe{}, err
+	}
+
+	for _, frame := range frames {
+		if frame.Current.Fn != nil && !strings.HasPrefix(frame.Current.Fn.Name, "runtime.") {
+			return frame, nil
+		}
 	}
+	return proc.Stackframe{}, fmt.Errorf("non-runtime frame not found")
+}
+
+func evalVariable(p proc.Process, symbol string, cfg proc.LoadConfig) (*proc.Variable, error) {
+	var scope *proc.EvalScope
+	var err error
+
+	if testBackend == "rr" {
+		var frame proc.Stackframe
+		frame, err = findFirstNonRuntimeFrame(p)
+		if err == nil {
+			scope = proc.FrameToScope(p, frame)
+		}
+	} else {
+		scope, err = proc.GoroutineScope(p.CurrentThread())
+	}
+
 	return scope.EvalVariable(symbol, cfg)
 }
 
@@ -83,11 +106,17 @@ func withTestProcess(name string, t *testing.T, fn func(p proc.Process, fixture
 	fixture := protest.BuildFixture(name)
 	var p proc.Process
 	var err error
+	var tracedir string
 	switch testBackend {
 	case "native":
 		p, err = native.Launch([]string{fixture.Path}, ".")
 	case "lldb":
 		p, err = gdbserial.LLDBLaunch([]string{fixture.Path}, ".")
+	case "rr":
+		protest.MustHaveRecordingAllowed(t)
+		t.Log("recording")
+		p, tracedir, err = gdbserial.RecordAndReplay([]string{fixture.Path}, ".", true)
+		t.Logf("replaying %q", tracedir)
 	default:
 		t.Fatalf("unknown backend %q", testBackend)
 	}
@@ -98,6 +127,9 @@ func withTestProcess(name string, t *testing.T, fn func(p proc.Process, fixture
 	defer func() {
 		p.Halt()
 		p.Detach(true)
+		if tracedir != "" {
+			protest.SafeRemoveAll(tracedir)
+		}
 	}()
 
 	fn(p, fixture)
@@ -148,6 +180,7 @@ func TestVariableEvaluation(t *testing.T) {
 		{"NonExistent", true, "", "", "", fmt.Errorf("could not find symbol value for NonExistent")},
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("testvariables", t, func(p proc.Process, fixture protest.Fixture) {
 		err := proc.Continue(p)
 		assertNoError(err, t, "Continue() returned an error")
@@ -166,7 +199,7 @@ func TestVariableEvaluation(t *testing.T) {
 				}
 			}
 
-			if tc.alternate != "" {
+			if tc.alternate != "" && testBackend != "rr" {
 				assertNoError(setVariable(p, tc.name, tc.alternate), t, "SetVariable()")
 				variable, err = evalVariable(p, tc.name, pnormalLoadConfig)
 				assertNoError(err, t, "EvalVariable()")
@@ -226,6 +259,7 @@ func TestVariableEvaluationShort(t *testing.T) {
 		{"NonExistent", true, "", "", "", fmt.Errorf("could not find symbol value for NonExistent")},
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("testvariables", t, func(p proc.Process, fixture protest.Fixture) {
 		err := proc.Continue(p)
 		assertNoError(err, t, "Continue() returned an error")
@@ -281,6 +315,7 @@ func TestMultilineVariableEvaluation(t *testing.T) {
 		Nest: *(*main.Nest)(…`, "", "main.Nest", nil},
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("testvariables", t, func(p proc.Process, fixture protest.Fixture) {
 		err := proc.Continue(p)
 		assertNoError(err, t, "Continue() returned an error")
@@ -354,13 +389,26 @@ func TestLocalVariables(t *testing.T) {
 				{"baz", true, "\"bazburzum\"", "", "string", nil}}},
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("testvariables", t, func(p proc.Process, fixture protest.Fixture) {
 		err := proc.Continue(p)
 		assertNoError(err, t, "Continue() returned an error")
 
 		for _, tc := range testcases {
-			scope, err := proc.GoroutineScope(p.CurrentThread())
-			assertNoError(err, t, "AsScope()")
+			var scope *proc.EvalScope
+			var err error
+
+			if testBackend == "rr" {
+				var frame proc.Stackframe
+				frame, err = findFirstNonRuntimeFrame(p)
+				if err == nil {
+					scope = proc.FrameToScope(p, frame)
+				}
+			} else {
+				scope, err = proc.GoroutineScope(p.CurrentThread())
+			}
+
+			assertNoError(err, t, "scope")
 			vars, err := tc.fn(scope, pnormalLoadConfig)
 			assertNoError(err, t, "LocalVariables() returned an error")
 
@@ -378,6 +426,7 @@ func TestLocalVariables(t *testing.T) {
 }
 
 func TestEmbeddedStruct(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		testcases := []varTest{
 			{"b.val", true, "-314", "-314", "int", nil},
@@ -654,6 +703,7 @@ func TestEvalExpression(t *testing.T) {
 		}
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		for _, tc := range testcases {
@@ -678,6 +728,7 @@ func TestEvalExpression(t *testing.T) {
 }
 
 func TestEvalAddrAndCast(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		c1addr, err := evalVariable(p, "&c1", pnormalLoadConfig)
@@ -704,6 +755,7 @@ func TestEvalAddrAndCast(t *testing.T) {
 }
 
 func TestMapEvaluation(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		m1v, err := evalVariable(p, "m1", pnormalLoadConfig)
@@ -738,6 +790,7 @@ func TestMapEvaluation(t *testing.T) {
 }
 
 func TestUnsafePointer(t *testing.T) {
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		up1v, err := evalVariable(p, "up1", pnormalLoadConfig)
@@ -775,6 +828,7 @@ func TestIssue426(t *testing.T) {
 
 	// Serialization of type expressions (go/ast.Expr) containing anonymous structs or interfaces
 	// differs from the serialization used by the linker to produce DWARF type information
+	protest.AllowRecording(t)
 	withTestProcess("testvariables2", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		for _, testcase := range testcases {
@@ -826,6 +880,7 @@ func TestPackageRenames(t *testing.T) {
 		return
 	}
 
+	protest.AllowRecording(t)
 	withTestProcess("pkgrenames", t, func(p proc.Process, fixture protest.Fixture) {
 		assertNoError(proc.Continue(p), t, "Continue() returned an error")
 		for _, tc := range testcases {