feat: Allow variables and stack trace inspection in the debugger (#4184)

# Description

## Problem\*

Part of #3015

Resolves #4025

## Summary\*

This PR adds several features to the debugger:
- (REPL) New command `stacktrace` to display the stack of the current
call frames.
- (REPL) New commands `over` and `out` to step to the next source code
location while staying without going into function calls or out of the
current stack frame.
- (DAP) The debugger commands "Step Into", "Step Out" and "Step Over"
now should work properly.
- (DAP) Return the stacktrace for the corresponding IDE panel.
- (DAP) Return the variables and Brillig registers for the corresponding
Variables panel in the IDE.

## Additional Context

## Documentation\*

Check one:
- [ ] No documentation needed.
- [ ] Documentation included in this PR.
- [X] **[Exceptional Case]** Documentation to be submitted in a separate
PR.

# PR Checklist\*

- [X] I have tested the changes locally.
- [X] I have formatted the changes with [Prettier](https://prettier.io/)
and/or `cargo fmt` on default settings.

acvm-repo/acvm/src/pwg/brillig.rs

@@ -130,6 +130,10 @@ impl<'b, B: BlackBoxFunctionSolver> BrilligSolver<'b, B> {
         self.vm.write_memory_at(ptr, value);
     }
 
+    pub fn get_call_stack(&self) -> Vec<usize> {
+        self.vm.get_call_stack()
+    }
+
     pub(super) fn solve(&mut self) -> Result<BrilligSolverStatus, OpcodeResolutionError> {
         let status = self.vm.process_opcodes();
         self.handle_vm_status(status)

acvm-repo/brillig_vm/src/lib.rs

@@ -167,6 +167,16 @@ impl<'a, B: BlackBoxFunctionSolver> VM<'a, B> {
         self.memory.write(MemoryAddress(ptr), value);
     }
 
+    /// Returns the VM's current call stack, including the actual program
+    /// counter in the last position of the returned vector.
+    pub fn get_call_stack(&self) -> Vec<usize> {
+        self.call_stack
+            .iter()
+            .map(|program_counter| program_counter.to_usize())
+            .chain(std::iter::once(self.program_counter))
+            .collect()
+    }
+
     /// Process a single opcode and modify the program counter.
     pub fn process_opcode(&mut self) -> VMStatus {
         let opcode = &self.bytecode[self.program_counter];

tooling/debugger/src/context.rs

@@ -78,6 +78,24 @@ impl<'a, B: BlackBoxFunctionSolver> DebugContext<'a, B> {
         }
     }
 
+    pub(super) fn get_call_stack(&self) -> Vec<OpcodeLocation> {
+        let instruction_pointer = self.acvm.instruction_pointer();
+        if instruction_pointer >= self.get_opcodes().len() {
+            vec![]
+        } else if let Some(ref solver) = self.brillig_solver {
+            solver
+                .get_call_stack()
+                .iter()
+                .map(|program_counter| OpcodeLocation::Brillig {
+                    acir_index: instruction_pointer,
+                    brillig_index: *program_counter,
+                })
+                .collect()
+        } else {
+            vec![OpcodeLocation::Acir(instruction_pointer)]
+        }
+    }
+
     pub(super) fn is_source_location_in_debug_module(&self, location: &Location) -> bool {
         self.debug_artifact
             .file_map
@@ -123,6 +141,21 @@ impl<'a, B: BlackBoxFunctionSolver> DebugContext<'a, B> {
             .unwrap_or(vec![])
     }
 
+    /// Returns the current call stack with expanded source locations. In
+    /// general, the matching between opcode location and source location is 1
+    /// to 1, but due to the compiler inlining functions a single opcode
+    /// location may expand to multiple source locations.
+    pub(super) fn get_source_call_stack(&self) -> Vec<(OpcodeLocation, Location)> {
+        self.get_call_stack()
+            .iter()
+            .flat_map(|opcode_location| {
+                self.get_source_location_for_opcode_location(opcode_location)
+                    .into_iter()
+                    .map(|source_location| (*opcode_location, source_location))
+            })
+            .collect()
+    }
+
     fn get_opcodes_sizes(&self) -> Vec<usize> {
         self.get_opcodes()
             .iter()
@@ -362,7 +395,8 @@ impl<'a, B: BlackBoxFunctionSolver> DebugContext<'a, B> {
         }
     }
 
-    pub(super) fn next(&mut self) -> DebugCommandResult {
+    /// Steps debugging execution until the next source location
+    pub(super) fn next_into(&mut self) -> DebugCommandResult {
         let start_location = self.get_current_source_location();
         loop {
             let result = self.step_into_opcode();
@@ -376,6 +410,38 @@ impl<'a, B: BlackBoxFunctionSolver> DebugContext<'a, B> {
         }
     }
 
+    /// Steps debugging execution until the next source location at the same (or
+    /// less) call stack depth (eg. don't dive into function calls)
+    pub(super) fn next_over(&mut self) -> DebugCommandResult {
+        let start_call_stack = self.get_source_call_stack();
+        loop {
+            let result = self.next_into();
+            if !matches!(result, DebugCommandResult::Ok) {
+                return result;
+            }
+            let new_call_stack = self.get_source_call_stack();
+            if new_call_stack.len() <= start_call_stack.len() {
+                return DebugCommandResult::Ok;
+            }
+        }
+    }
+
+    /// Steps debugging execution until the next source location with a smaller
+    /// call stack depth (eg. returning from the current function)
+    pub(super) fn next_out(&mut self) -> DebugCommandResult {
+        let start_call_stack = self.get_source_call_stack();
+        loop {
+            let result = self.next_into();
+            if !matches!(result, DebugCommandResult::Ok) {
+                return result;
+            }
+            let new_call_stack = self.get_source_call_stack();
+            if new_call_stack.len() < start_call_stack.len() {
+                return DebugCommandResult::Ok;
+            }
+        }
+    }
+
     pub(super) fn cont(&mut self) -> DebugCommandResult {
         loop {
             let result = self.step_into_opcode();

tooling/debugger/src/dap.rs

@@ -18,12 +18,12 @@ use dap::requests::{Command, Request, SetBreakpointsArguments};
 use dap::responses::{
     ContinueResponse, DisassembleResponse, ResponseBody, ScopesResponse, SetBreakpointsResponse,
     SetExceptionBreakpointsResponse, SetInstructionBreakpointsResponse, StackTraceResponse,
-    ThreadsResponse,
+    ThreadsResponse, VariablesResponse,
 };
 use dap::server::Server;
 use dap::types::{
-    Breakpoint, DisassembledInstruction, Source, StackFrame, SteppingGranularity,
-    StoppedEventReason, Thread,
+    Breakpoint, DisassembledInstruction, Scope, Source, StackFrame, SteppingGranularity,
+    StoppedEventReason, Thread, Variable,
 };
 use nargo::artifacts::debug::DebugArtifact;
 
@@ -41,6 +41,22 @@ pub struct DapSession<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> {
     source_breakpoints: BTreeMap<FileId, Vec<(OpcodeLocation, i64)>>,
 }
 
+enum ScopeReferences {
+    Locals = 1,
+    WitnessMap = 2,
+    InvalidScope = 0,
+}
+
+impl From<i64> for ScopeReferences {
+    fn from(value: i64) -> Self {
+        match value {
+            1 => Self::Locals,
+            2 => Self::WitnessMap,
+            _ => Self::InvalidScope,
+        }
+    }
+}
+
 // BTreeMap<FileId, Vec<(usize, OpcodeLocation)>
 
 impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
@@ -132,7 +148,7 @@ impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
             // source location to show when first starting the debugger, but
             // maybe the default behavior should be to start executing until the
             // first breakpoint set.
-            _ = self.context.next();
+            _ = self.context.next_into();
         }
 
         self.server.send_event(Event::Initialized)?;
@@ -176,34 +192,33 @@ impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
                         args.granularity.as_ref().unwrap_or(&SteppingGranularity::Statement);
                     match granularity {
                         SteppingGranularity::Instruction => self.handle_step(req)?,
-                        _ => self.handle_next(req)?,
+                        _ => self.handle_next_into(req)?,
                     }
                 }
                 Command::StepOut(ref args) => {
                     let granularity =
                         args.granularity.as_ref().unwrap_or(&SteppingGranularity::Statement);
                     match granularity {
                         SteppingGranularity::Instruction => self.handle_step(req)?,
-                        _ => self.handle_next(req)?,
+                        _ => self.handle_next_out(req)?,
                     }
                 }
                 Command::Next(ref args) => {
                     let granularity =
                         args.granularity.as_ref().unwrap_or(&SteppingGranularity::Statement);
                     match granularity {
                         SteppingGranularity::Instruction => self.handle_step(req)?,
-                        _ => self.handle_next(req)?,
+                        _ => self.handle_next_over(req)?,
                     }
                 }
                 Command::Continue(_) => {
                     self.handle_continue(req)?;
                 }
                 Command::Scopes(_) => {
-                    // FIXME: this needs a proper implementation when we can
-                    // show the parameters and variables
-                    self.server.respond(
-                        req.success(ResponseBody::Scopes(ScopesResponse { scopes: vec![] })),
-                    )?;
+                    self.handle_scopes(req)?;
+                }
+                Command::Variables(ref _args) => {
+                    self.handle_variables(req)?;
                 }
                 _ => {
                     eprintln!("ERROR: unhandled command: {:?}", req.command);
@@ -213,37 +228,38 @@ impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
         Ok(())
     }
 
+    fn build_stack_trace(&self) -> Vec<StackFrame> {
+        self.context
+            .get_source_call_stack()
+            .iter()
+            .enumerate()
+            .map(|(index, (opcode_location, source_location))| {
+                let line_number =
+                    self.debug_artifact.location_line_number(*source_location).unwrap();
+                let column_number =
+                    self.debug_artifact.location_column_number(*source_location).unwrap();
+                StackFrame {
+                    id: index as i64,
+                    name: format!("frame #{index}"),
+                    source: Some(Source {
+                        path: self.debug_artifact.file_map[&source_location.file]
+                            .path
+                            .to_str()
+                            .map(String::from),
+                        ..Source::default()
+                    }),
+                    line: line_number as i64,
+                    column: column_number as i64,
+                    instruction_pointer_reference: Some(opcode_location.to_string()),
+                    ..StackFrame::default()
+                }
+            })
+            .rev()
+            .collect()
+    }
+
     fn handle_stack_trace(&mut self, req: Request) -> Result<(), ServerError> {
-        let opcode_location = self.context.get_current_opcode_location();
-        let source_location = self.context.get_current_source_location();
-        let frames = match source_location {
-            None => vec![],
-            Some(locations) => locations
-                .iter()
-                .enumerate()
-                .map(|(index, location)| {
-                    let line_number = self.debug_artifact.location_line_number(*location).unwrap();
-                    let column_number =
-                        self.debug_artifact.location_column_number(*location).unwrap();
-                    let ip_reference = opcode_location.map(|location| location.to_string());
-                    StackFrame {
-                        id: index as i64,
-                        name: format!("frame #{index}"),
-                        source: Some(Source {
-                            path: self.debug_artifact.file_map[&location.file]
-                                .path
-                                .to_str()
-                                .map(String::from),
-                            ..Source::default()
-                        }),
-                        line: line_number as i64,
-                        column: column_number as i64,
-                        instruction_pointer_reference: ip_reference,
-                        ..StackFrame::default()
-                    }
-                })
-                .collect(),
-        };
+        let frames = self.build_stack_trace();
         let total_frames = Some(frames.len() as i64);
         self.server.respond(req.success(ResponseBody::StackTrace(StackTraceResponse {
             stack_frames: frames,
@@ -315,9 +331,23 @@ impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
         self.handle_execution_result(result)
     }
 
-    fn handle_next(&mut self, req: Request) -> Result<(), ServerError> {
-        let result = self.context.next();
-        eprintln!("INFO: stepped by statement with result {result:?}");
+    fn handle_next_into(&mut self, req: Request) -> Result<(), ServerError> {
+        let result = self.context.next_into();
+        eprintln!("INFO: stepped into by statement with result {result:?}");
+        self.server.respond(req.ack()?)?;
+        self.handle_execution_result(result)
+    }
+
+    fn handle_next_out(&mut self, req: Request) -> Result<(), ServerError> {
+        let result = self.context.next_out();
+        eprintln!("INFO: stepped out by statement with result {result:?}");
+        self.server.respond(req.ack()?)?;
+        self.handle_execution_result(result)
+    }
+
+    fn handle_next_over(&mut self, req: Request) -> Result<(), ServerError> {
+        let result = self.context.next_over();
+        eprintln!("INFO: stepped over by statement with result {result:?}");
         self.server.respond(req.ack()?)?;
         self.handle_execution_result(result)
     }
@@ -548,6 +578,73 @@ impl<'a, R: Read, W: Write, B: BlackBoxFunctionSolver> DapSession<'a, R, W, B> {
         )?;
         Ok(())
     }
+
+    fn handle_scopes(&mut self, req: Request) -> Result<(), ServerError> {
+        self.server.respond(req.success(ResponseBody::Scopes(ScopesResponse {
+            scopes: vec![
+                Scope {
+                    name: String::from("Locals"),
+                    variables_reference: ScopeReferences::Locals as i64,
+                    ..Scope::default()
+                },
+                Scope {
+                    name: String::from("Witness Map"),
+                    variables_reference: ScopeReferences::WitnessMap as i64,
+                    ..Scope::default()
+                },
+            ],
+        })))?;
+        Ok(())
+    }
+
+    fn build_local_variables(&self) -> Vec<Variable> {
+        let mut variables: Vec<_> = self
+            .context
+            .get_variables()
+            .iter()
+            .map(|(name, value, _var_type)| Variable {
+                name: String::from(*name),
+                value: format!("{:?}", *value),
+                ..Variable::default()
+            })
+            .collect();
+        variables.sort_by(|a, b| a.name.partial_cmp(&b.name).unwrap());
+        variables
+    }
+
+    fn build_witness_map(&self) -> Vec<Variable> {
+        self.context
+            .get_witness_map()
+            .clone()
+            .into_iter()
+            .map(|(witness, value)| Variable {
+                name: format!("_{}", witness.witness_index()),
+                value: format!("{value:?}"),
+                ..Variable::default()
+            })
+            .collect()
+    }
+
+    fn handle_variables(&mut self, req: Request) -> Result<(), ServerError> {
+        let Command::Variables(ref args) = req.command else {
+            unreachable!("handle_variables called on a different request");
+        };
+        let scope: ScopeReferences = args.variables_reference.into();
+        let variables: Vec<_> = match scope {
+            ScopeReferences::Locals => self.build_local_variables(),
+            ScopeReferences::WitnessMap => self.build_witness_map(),
+            _ => {
+                eprintln!(
+                    "handle_variables with an unknown variables_reference {}",
+                    args.variables_reference
+                );
+                vec![]
+            }
+        };
+        self.server
+            .respond(req.success(ResponseBody::Variables(VariablesResponse { variables })))?;
+        Ok(())
+    }
 }
 
 pub fn run_session<R: Read, W: Write, B: BlackBoxFunctionSolver>(