Merge pull request #166 from puremourning/debugpy-remote

Debugpy remote debugging and install issues

README.md

@@ -793,11 +793,25 @@ An alternative is to to use `lldb-vscode`, which comes with llvm.  Here's how:
 }
 ```
 
+### Remote debugging
+
+The cpptools documentation describes how to attach cpptools to gdbserver using
+`miDebuggerAddress`. Note that when doing this you should use the
+`"request": "attach"`.
+
+### Remote launch and attach
+
+If you're feeling fancy, checkout the [reference guide][remote-debugging] for
+an example of getting Vimspector to remotely launch and attach.
+
 ## Python
 
 * Python: [debugpy][]
-* Requires `install_gadget.py --enable-python`, this requires a working compiler
+* Requires `install_gadget.py --enable-python`, ideally requires a working
-to build a C python extension for performance.
+  compiler and the python development headers/libs to build a C python extension
+  for performance.
+* Full options: https://github.com/microsoft/debugpy/wiki/Debug-configuration-settings
+
 
 **Migrating from `vscode-python`**: change `"adapter": "vscode-python"` to
 `"adapter": "debugpy"`.
@@ -816,7 +830,7 @@ to build a C python extension for performance.
         "stopOnEntry": true,
         "console": "externalTerminal",
         "debugOptions": [],
-        "program": "<path to main python file>",
+        "program": "<path to main python file>"
       }
     }
     ...
@@ -824,6 +838,50 @@ to build a C python extension for performance.
 }
 ```
 
+### Remote Debugging
+
+In order to use remote debugging with debugpy, you have to connect Vimspector
+directly to the application that is being debugged. This is easy, but it's a
+little different from how we normally configure things. Specifically, you need
+to:
+
+
+* Start your application with debugpy, specifying the `--listen` argument. See
+  [the debugpy
+  documentation](https://github.com/microsoft/debugpy#debugpy-cli-usage) for
+  details.
+* use the built-in "multi-session" adapter. This just asks for the host/port to
+  connect to. For example:
+
+```json
+{
+  "configurations": {
+    "Python Attach": {
+      "adapter": "multi-session",
+      "configuration": {
+        "request": "attach",
+        "pathMappings": [
+          // mappings here (optional)
+        ]
+      }
+    }
+  }
+}
+```
+
+See [deatils of the launch
+configuration](https://github.com/microsoft/debugpy/wiki/Debug-configuration-settings)
+for explanation of things like `pathMappings`.
+
+Additional documenation, including how to do this when the remote machine can
+only be contacted via SSH [are provided by
+debugpy](https://github.com/microsoft/debugpy/wiki/Debugging-over-SSH).
+
+### Remote launch and attach
+
+If you're feeling fancy, checkout the [reference guide][remote-debugging] for
+an example of getting Vimspector to remotely launch and attach.
+
 ### Legacy: vscode-python
 
 * No longer installed by default - please pass `--force-enable-python.legacy` if
@@ -884,7 +942,7 @@ Requires `install_gadget.py --force-enable-csharp`
 
 Requires `install_gadget.py --force-enable-csharp`.
 
-***Known not to work.****
+***Known not to work.***
 
 ```json
 {
@@ -1093,7 +1151,7 @@ use it with Vimspector.
         "port": "${port}",
         "sourcePaths": [
           "${workspaceRoot}/src/main/java",
-          "${workspaceRoot}/src/test/java",
+          "${workspaceRoot}/src/test/java"
         ]
       }
     }
@@ -1151,6 +1209,7 @@ background.
   comment](https://github.com/puremourning/vimspector/issues/3#issuecomment-576916076)
   for instructions.
 
+
 # Customisation
 
 There is very limited support for customistaion of the UI.
@@ -1186,6 +1245,15 @@ sign define vimspectorPC text=🔶 texthl=SpellBad
    in my `.vimspector.json` ? Yes, see [here][vimspector-ref-exception].
 5. Do I have to specify the file to execute in `.vimspector.json`, or could it be the current vim file? 
    You don't need to. You can specify $file for the current active file. See [here][vimspector-ref-var] for complete list of replacements in the configuration file.
+6. You allow comments in `.vimspector.json`, but Vim highlights these as errors,
+   do you know how to make this not-an-error? Yes, put this in
+   `~/.vim/after/syntax/json.vim`:
+
+```viml
+syn region jsonComment start="/\*" end="\*/"
+hi link jsonCommentError Comment
+hi link jsonComment Comment
+```
 
 # License
 
@@ -1204,3 +1272,4 @@ Copyright © 2018 Ben Jackson
 [vimspector-ref-exception]: https://puremourning.github.io/vimspector/configuration.html#exception-breakpoints
 [debugpy]: https://github.com/microsoft/debugpy
 [YouCompleteMe]: https://github.com/ycm-core/YouCompleteMe#java-semantic-completion
+[remote-debugging]: https://puremourning.github.io/vimspector/configuration.html#remote-debugging-support

autoload/vimspector/internal/channel.vim

@@ -39,7 +39,7 @@ function! vimspector#internal#channel#StartDebugSession( config ) abort
     return v:false
   endif
 
-  let l:addr = 'localhost:' . a:config[ 'port' ]
+  let l:addr = get( a:config, 'host', 'localhost' ) . ':' . a:config[ 'port' ]
 
   echo 'Connecting to ' . l:addr . '... (waiting fo up to 10 seconds)'
   let s:ch = ch_open( l:addr,

autoload/vimspector/internal/neochannel.vim

@@ -39,7 +39,7 @@ function! vimspector#internal#neochannel#StartDebugSession( config ) abort
     return v:false
   endif
 
-  let addr = 'localhost:' . a:config[ 'port' ]
+  let l:addr = get( a:config, 'host', 'localhost' ) . ':' . a:config[ 'port' ]
 
   let s:ch = sockconnect( 'tcp', addr, { 'on_data': funcref( 's:_OnEvent' ) } )
   if s:ch <= 0

compiler/vimspector_test.vim

@@ -90,7 +90,7 @@ function! s:RunTestUnderCursor()
   let l:cwd = getcwd()
   execute 'lcd ' . s:root_dir
   try
-    execute s:make_cmd . ' ' . l:test_arg
+    execute s:make_cmd . ' ' . get( b:, 'test_args', '' ) . ' ' . l:test_arg
   finally
     execute 'lcd ' . l:cwd
   endtry
@@ -101,7 +101,7 @@ function! s:RunTest()
   let l:cwd = getcwd()
   execute 'lcd ' . s:root_dir
   try
-    execute s:make_cmd . ' %:p:t'
+    execute s:make_cmd . ' ' . get( b:, 'test_args', '' ) . ' %:p:t'
   finally
     execute 'lcd ' . l:cwd
   endtry
@@ -112,7 +112,7 @@ function! s:RunAllTests()
   let l:cwd = getcwd()
   execute 'lcd ' . s:root_dir
   try
-    execute s:make_cmd
+    execute s:make_cmd . ' ' . get( b:, 'test_args', '' )
   finally
     execute 'lcd ' . l:cwd
   endtry

docs/configuration.md

@@ -365,6 +365,251 @@ The following variables are provided:
 * `${fileExtname}` - the current opened file's extension
 * `${cwd}` - the current working directory of the active window on launch
 
+## Remote Debugging Support
+
+Vimspector has in-built support for exectuting remote debuggers (such as
+`gdbserver`, `debugpy`, `llvm-server` etc.). This is useful for environments
+where the development is done on one host and the runtime is
+some other host, account, container, etc.
+
+In order for it to work, you have to set up passwordless SSH between the local
+and remote machines/accounts. Then just tell Vimsector how to remotely launch
+and/or attach to the app.
+
+This is presented as examples with commentary, as it's a fairly advanced/niche
+case. If you're not already familiar with remote debugging tools (such as
+gdbserver) or not familar with ssh or such, you might need to independently
+research that.
+
+Vimspector's tools are intended to automate your existing process for setting
+this up rather than to offer batteries-included approach. Ultimately, all
+vimspector is going to do is run your commands over SSH and co-ordinate with the
+adapter.
+
+### Python (debugpy) Example
+
+Here is some examples using the vimspector built-in
+remote support (using SSH) to remotely launch and attach a python application
+and connect to it using debugpy.
+
+The usage pattern is to hit `<F5>`, enter `host` (the host where your app runs),
+`account` (the account it runs under), and `port` (a port that will be opened on
+the remote host). Vimspector then orchestrates the various tools to set you up.
+
+```json
+
+{
+  "adapters": {
+    "python-remote": {
+      "port": "${port}",
+      "host": "${host}",
+      "launch": {
+        "remote": {
+          "host": "${host}",       // Remote host to ssh to (mandatory)
+          "account": "${account}", // User to connect as (optional)
+
+          // Optional.... Manual additional arguments for ssh
+          // "ssh": {
+          //   "args": [ "-o", "StrictHostKeyChecking=no" ]
+          // },
+
+          // Command to launch the debugee and attach the debugger; 
+          // %CMD% replaced with the remote-cmdLine configured in the launch
+          // configuration. (mandatory)
+          "launchCommmand": [
+            "python", "-m", "debugpy", "--listen 0.0.0.0:${port}",
+            "%CMD%"
+          ]
+          
+          // Optional alternative to launchCommmand (if you need to run multiple
+          // commands)
+          // "launchCommmands":  [
+          //   [ /* first command */ ],
+          //   [ /* second command */ ]
+          // ]
+
+        }
+      },
+      "attach": {
+        "remote": {
+          "host": "${host}",       // Remote host to ssh to (mandatory)
+          "account": "${account}", // User to connect as (optional)
+
+          // Optional.... Manual additional arguments for ssh
+          // "ssh": {
+          //   "args": [ "-o", "StrictHostKeyChecking=no" ]
+          // },
+
+          // Command to get the PID of the process to attach  (mandatory)
+          "pidCommand": [
+            //
+            // Remember taht you can use ${var} to ask for input. I use this to
+            // call a custom command to returm the PID for a named service, so
+            // here's an examle:
+            //
+            "/path/to/secret/script/GetPIDForService", "${ServiceName}"
+          ],
+
+          // Command to attach the debugger; %PID% replaced with output of
+          // pidCommand above (mandatory)
+          "attachCommand": [
+            "python", "-m", "debugpy", "--listen 0.0.0.0:${port}",
+            "--pid", "%PID%"
+          ]
+          
+          // Optional alternative to attachCommand (if you need to run multiple
+          // commands)
+          // "attachCommands":  [
+          //   [ /* first command */ ],
+          //   [ /* second command */ ]
+          // ],
+
+          // Optional.... useful with buggy gdbservers to kill -TRAP %PID%
+          // "initCompleteCommand": [
+          //   /* optional command to run after initialized */
+          // ]
+
+          // Optional.... Manual additional arguments for ssh
+          // "ssh": {
+          //   "args": [ "-o", "StrictHostKeyChecking=no" ]
+          // },
+        }
+      }
+    }
+  },
+  "configurations": {
+    "remote-launch": {
+      "adapter": "python-remote",
+
+      "remote-request": "launch",
+      "remote-cmdLine": [
+        "${RemoteRoot}/${fileBasename}", "*${args}"
+      ],
+
+      "configuration": {
+        "request": "attach",
+        "pathMappings": [
+          {
+            "localRoot": "${workspaceRoot}",
+            "remoteRoot": "${RemoteRoot}"
+          }
+        ]
+      }
+    },
+    "remote-attach": {
+      "variables": {
+        // Just an example of how to specify a variable manually rather than
+        // vimspector asking for input from the user
+        "ServiceName": "${fileBasenameNoExtention}"
+      },
+
+      "adapter": "python-remote",
+      "remote-request": "attach",
+
+      "configuration": {
+        "request": "attach",
+        "pathMappings": [
+          {
+            "localRoot": "${workspaceRoot}",
+            "remoteRoot": "${RemoteRoot}"
+          }
+        ]
+      }
+    }
+  }
+}
+```
+
+### C-family (gdbserver) Example
+
+This example uses vimspector to remotely luanch or attach to a binary using
+`gdbserver` and then instructs vscode-cpptools to attach to that `gdbserver`.
+
+The appraoch is very similar to the above for python, just that we use gdbserver
+and have to tell cpptools a few more options.
+
+```json
+{
+  "adapters": {
+    "cpptools-remote": {
+      "command": [
+        "${gadgetDir}/vscode-cpptools/debugAdapters/OpenDebugAD7"
+      ],
+      "name": "cppdbg",
+      "configuration": {
+        "type": "cppdbg"
+      },
+      "launch": {
+        "remote": {
+          "host": "${host}",
+          "account": "${account}",
+          "launchCommmand": [ 
+            "gdbserver",
+            "--once",
+            "--no-startup-with-shell",
+            "--disable-randomisation",
+            "0.0.0.0:${port}",
+            "%CMD%"
+        }
+      },
+      "attach": {
+        "remote": {
+          "host": "${host}",
+          "account": "${account}",
+          "pidCommand": [
+            "/path/to/secret/script/GetPIDForService", "${ServiceName}"
+          ],
+          "attachCommand": [ 
+            "gdbserver",
+            "--once",
+            "--attach",
+            "0.0.0.0:${port}",
+            "%PID%"
+          ],
+          //
+          // If your application is started by a wrapper script, then you might
+          // need the followin. GDB can't pause an application because it only
+          // sends the signal to the process group leader. Or something.
+          // Basically, if you find that everything just hangs and the
+          // application never attaches, try using the following to manually
+          // force the trap signal.
+          //
+          "initCompleteCommand": [
+            "kill",
+            "-TRAP",
+            "%PID%"
+          ]
+        }
+      }
+    }
+  },
+  "configurations": {
+    "remote launch": {
+      "adapter": "cpptools-remote",
+      "remote-cmdLine": [ "/path/to/the/remote/executable", "args..." ],
+      "remote-request": "launch",
+      "configuration": {
+        "request": "attach", // yes, attach!
+        
+        "program": "/path/to/the/local/executable",
+        "MIMode": "gdb",
+        "miDebuggerAddress": "${host}:${port}"
+      }
+    },
+    "remote attach": {
+      "adapter": "cpptools-remote",
+      "remote-request": "attach",
+      "configuration": {
+        "request": "attach",
+        
+        "program": "/path/to/the/local/executable",
+        "MIMode": "gdb",
+        "miDebuggerAddress": "${host}:${port}"
+    }
+  }
+}
+```
+
 ## Appendix: Configuration file format
 
 The configuration files are text files which must be UTF-8 encoded. They

install_gadget.py

@@ -73,6 +73,12 @@ GADGETS = {
           "pidProperty": "processId",
           "pidSelect": "ask"
         },
+        "configuration": {
+          "type": "cppdbg",
+          "args": [],
+          "cwd": "${workspaceRoot}",
+          "environment": [],
+        }
       },
     },
   },
@@ -576,6 +582,11 @@ parser.add_argument( '--no-gadget-config',
                      action = 'store_true',
                      help = "Don't write the .gagets.json, just install" )
 
+parser.add_argument( '--update-gadget-config',
+                     action = 'store_true',
+                     help =
+                       "Update the gadget config rather than overwrite it" )
+
 parser.add_argument( '--enable-custom',
                      dest='custom_gadget_file',
                      action='append',
@@ -657,7 +668,20 @@ for custom_file_name in functools.reduce( operator.add,
 
 
 failed = []
-all_adapters = {}
+if args.update_gadget_config:
+  with open( install.GetGadgetConfigFile( vimspector_base ), 'r' ) as f:
+    all_adapters = json.load( f ).get( 'adapters', {} )
+else:
+  all_adapters = {}
+
+# Include "built-in" adapter for multi-session mode
+all_adapters.update( {
+  'multi-session': {
+    'port': '${port}',
+    'host': '${host}'
+  },
+} )
+
 for name, gadget in GADGETS.items():
   if not gadget.get( 'enabled', True ):
     if ( not args.force_all

python3/vimspector/installer.py

@@ -140,13 +140,34 @@ def ValidateCheckSumSHA256( file_path, checksum ):
 
 
 def RemoveIfExists( destination ):
-  if os.path.exists( destination ) or os.path.islink( destination ):
+  if os.path.islink( destination ):
-    if os.path.islink( destination ):
+    print( "Removing file {}".format( destination ) )
-      print( "Removing file {}".format( destination ) )
+    os.remove( destination )
-      os.remove( destination )
+    return
-    else:
+
-      print( "Removing dir {}".format( destination ) )
+  N = 1
+
+
+  def BackupDir():
+    return "{}.{}".format( destination, N )
+
+  while os.path.isdir( BackupDir() ):
+    print( "Removing old dir {}".format( BackupDir() ) )
+    try:
+      shutil.rmtree( BackupDir() )
+      print ( "OK, removed it" )
+      break
+    except OSError:
+      print ( "FAILED" )
+      N = N + 1
+
+  if os.path.exists( destination ):
+    print( "Removing dir {}".format( destination ) )
+    try:
       shutil.rmtree( destination )
+    except OSError:
+      print( "FAILED, moving {} to dir {}".format( destination, BackupDir() ) )
+      os.rename( destination, BackupDir() )
 
 
 # Python's ZipFile module strips execute bits from files, for no good reason

support/test/java/test_project/pom.xml

@@ -4,7 +4,7 @@
   <artifactId>TestApplication</artifactId>
   <version>1</version>
   <properties>
-    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>1.8</maven.compiler.target>
+    <maven.compiler.target>11</maven.compiler.target>
   </properties>
 </project>

support/test/python/simple_python/.vimspector.json

@@ -2,7 +2,7 @@
   "configurations": {
     // This is a comment.
     "run legacy vscode-python": {
-      "adapter": "vscode-python" /* coment goes here too */,
+      "adapter": "vscode-python", /* coment goes here too */
       "configuration": {
         "request": "launch",
         "type": "python",
@@ -19,12 +19,9 @@
       }
     },
     "attach": {
-      "adapter": "debugpy",
+      "adapter": "multi-session",
       "configuration": {
-        "request": "attach",
+        "request": "attach"
-        "type": "python",
-        "host": "localhost",
-        "port": "5678"
       },
       "breakpoints": {
         "exception": {

support/test/python/simple_python/requirements.txt

@@ -1 +1 @@
-ptvsd==4.3.2
+debugpy

support/test/python/simple_python/run_with_debugpy

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+PYTHON=python3
+
+if [ -n "$1" ]; then
+  PYTHON=$1
+fi
+
+pushd $(dirname $0)
+  if [ -d env ]; then
+    rm -rf env
+  fi
+
+  $PYTHON -m venv env
+  . env/bin/activate
+  python -m pip install -r requirements.txt
+  echo "*** Launching ***"
+  python -m debugpy --wait-for-client --listen 0.0.0.0:5678 main.py
+popd
+

support/test/python/simple_python/run_with_ptvsd

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-PYTHON=python3
-
-if [ -n "$1" ]; then
-  PYTHON=$1
-fi
-
-pushd $(dirname $0)
-  if [ ! -d env ]; then
-    virtualenv -p $PYTHON env
-  fi
-
-  . env/bin/activate
-  pip install -r requirements.txt
-
-  python -m ptvsd --wait --host localhost --port 5678 main.py
-popd
-

tests/ci/image/Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update && \
   apt-get -y dist-upgrade && \
   apt-get -y install python3-dev \
                      python3-pip \
+                     python3-venv \
                      ca-cacert \
                      locales \
                      language-pack-en \

tests/language_python.test.vim

@@ -45,3 +45,63 @@ function! Test_Python_Simple()
   %bwipeout!
 endfunction
 
+function! SetUp_Test_Python_Remote_Attach()
+  let g:vimspector_enable_mappings = 'HUMAN'
+endfunction
+
+function! Test_Python_Remote_Attach()
+  lcd ../support/test/python/simple_python
+  let fn='main.py'
+  exe 'edit ' . fn
+
+  let ready = v:false
+  function! ReceiveFromLauncher( ch, data ) closure
+    if a:data ==# '*** Launching ***'
+      let ready = v:true
+    endif
+  endfunction
+
+  let jobid = job_start( [ './run_with_debugpy' ], {
+        \ 'out_mode': 'nl',
+        \ 'out_cb': funcref( 'ReceiveFromLauncher' ),
+        \ } )
+
+  " Wait up to 60s for the debugee to be launched (the script faffs with
+  " virtualenvs etc.)
+  call WaitFor( {-> ready == v:true }, 60000 )
+
+  call setpos( '.', [ 0, 6, 1 ] )
+
+  call vimspector#test#signs#AssertCursorIsAtLineInBuffer( fn, 6, 1 )
+  call vimspector#test#signs#AssertSignGroupEmptyAtLine( 'VimspectorBP', 6 )
+
+  " Add the breakpoint
+  call feedkeys( "\<F9>", 'xt' )
+  call vimspector#test#signs#AssertSignGroupSingletonAtLine( 'VimspectorBP',
+                                                           \ 6,
+                                                           \ 'vimspectorBP' )
+
+  call setpos( '.', [ 0, 1, 1 ] )
+
+  " Here we go. Start Debugging (note will wait up to 10s for the script to do
+  " its virtualenv thing)
+  call vimspector#LaunchWithSettings( {
+        \   'configuration': 'attach',
+        \   'port': 5678,
+        \   'host': 'localhost'
+        \ } )
+  call vimspector#test#signs#AssertCursorIsAtLineInBuffer( fn, 6, 1 )
+
+  " Step
+  call feedkeys( "\<F10>", 'xt' )
+
+  call vimspector#test#signs#AssertCursorIsAtLineInBuffer( fn, 7, 1 )
+  call WaitForAssert( {->
+        \ vimspector#test#signs#AssertPCIsAtLineInBuffer( fn, 7 )
+        \ } )
+
+  call vimspector#test#setup#Reset()
+  call job_stop( jobid )
+  lcd -
+  %bwipeout!
+endfunction

tests/testdata/cpp/simple/.vimspector.json

@@ -3,12 +3,8 @@
     "cpptools-run": {
       "adapter": "vscode-cpptools",
       "configuration": {
-        "type": "cppdbg",
         "request": "launch",
         "program": "${workspaceRoot}/${fileBasenameNoExtension}",
-        "args": [],
-        "cwd": "${workspaceRoot}",
-        "environment": [],
         "externalConsole": false,
         "stopAtEntry": true,
         "stopOnEntry": true,