Add offset support, sort of

.github/workflows/build.yaml

@@ -13,7 +13,7 @@ defaults:
 
 jobs:
   PythonLint:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-16.04
     container: 'puremourning/vimspector:test'
     steps:
     - uses: actions/checkout@v2
@@ -22,7 +22,7 @@ jobs:
     - name: 'Run flake8'
       run: '$HOME/.local/bin/flake8 python3/ *.py'
   VimscriptLint:
-    runs-on: 'ubuntu-18.04'
+    runs-on: 'ubuntu-16.04'
     container: 'puremourning/vimspector:test'
     steps:
     - uses: actions/checkout@v2
@@ -32,7 +32,7 @@ jobs:
       run: $HOME/.local/bin/vint autoload/ compiler/ plugin/ tests/ syntax/
 
   Linux:
-    runs-on: 'ubuntu-18.04'
+    runs-on: 'ubuntu-16.04'
     container:
       image: 'puremourning/vimspector:test'
       options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
@@ -156,7 +156,7 @@ jobs:
     #     SSH_PASS: ${{ secrets.SSH_PASS }} # [V]imspector
 
   PublishRelease:
-    runs-on: 'ubuntu-18.04'
+    runs-on: 'ubuntu-16.04'
     needs:
       - Linux
       - MacOS

README.md

@@ -61,10 +61,9 @@ For detailed explanation of the `.vimspector.json` format, see the
     * [Closing debugger](#closing-debugger)
     * [Terminate debuggee](#terminate-debuggee)
  * [Debug profile configuration](#debug-profile-configuration)
-    * [C, C++, Rust, etc.](#c-c-rust-etc)
-       * [Data visualization / pretty printing](#data-visualization--pretty-printing)
-       * [C++ Remote debugging](#c-remote-debugging)
-       * [C++ Remote launch and attach](#c-remote-launch-and-attach)
+    * [C, C  , Rust, etc.](#c-c-rust-etc)
+       * [C   Remote debugging](#c-remote-debugging)
+       * [C   Remote launch and attach](#c-remote-launch-and-attach)
     * [Rust](#rust)
     * [Python](#python)
        * [Python Remote Debugging](#python-remote-debugging)
@@ -291,7 +290,7 @@ If you just want to try out vimspector without changing your vim config, there
 are example projects for a number of languages in `support/test`, including:
 
 * Python (`support/test/python/simple_python`)
-* Go (`support/test/go/hello_world` and `support/test/go/name-starts-with-vowel`)
+* Go (`support/test/go/hello_world`)
 * Nodejs (`support/test/node/simple`)
 * Chrome (`support/test/chrome/`)
 * etc.
@@ -1177,38 +1176,6 @@ licensing.
 }
 ```
 
-### Data visualization / pretty printing
-
-Depending on the backend you need to enable pretty printing of complex types manually.
-
-* LLDB: Pretty printing is enabled by default
-
-* GDB: To enable gdb pretty printers, consider the snippet below.  
-  It is not enough to have `set print pretty on` in your .gdbinit!
-
-```
-{
-  "configurations": {
-    "Launch": {
-      "adapter": "vscode-cpptools",
-      "configuration": {
-        "request": "launch",
-        "program": "<path to binary>",
-        ...
-        "MIMode": "gdb"
-        "setupCommands": [
-          {
-            "description": "Enable pretty-printing for gdb",
-            "text": "-enable-pretty-printing",
-            "ignoreFailures": true
-          }
-        ],
-      }
-    }
-  }
-}
-```
-
 ### C++ Remote debugging
 
 The cpptools documentation describes how to attach cpptools to gdbserver using
@@ -1401,8 +1368,6 @@ Requires:
 * [Delve][delve-install] installed, e.g. `go get -u github.com/go-delve/delve/cmd/dlv`
 * Delve to be in your PATH, or specify the `dlvToolPath` launch option
 
-NOTE: Vimspector uses the ["legacy" vscode-go debug adapter](https://github.com/golang/vscode-go/blob/master/docs/debugging-legacy.md) rather than the "built-in" DAP support in Delve. You can track https://github.com/puremourning/vimspector/issues/186 for that.
-
 ```json
 {
   "configurations": {
@@ -1420,7 +1385,7 @@ NOTE: Vimspector uses the ["legacy" vscode-go debug adapter](https://github.com/
 ```
 
 See the vscode-go docs for
-[troubleshooting information](https://github.com/golang/vscode-go/blob/master/docs/debugging-legacy.md#troubleshooting)
+[troubleshooting information](https://github.com/golang/vscode-go/blob/master/docs/debugging.md#troubleshooting)
 
 ## PHP
 

autoload/vimspector.vim

@@ -223,6 +223,13 @@ function! vimspector#SetVariableValue( ... ) abort
   endif
 endfunction
 
+function! vimspector#ReadMemory() abort
+  if !s:Enabled()
+    return
+  endif
+  py3 _vimspector_session.ReadMemory()
+endfunction
+
 function! vimspector#DeleteWatch() abort
   if !s:Enabled()
     return

docs/Gemfile.lock

@@ -7,7 +7,7 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
-    addressable (2.8.0)
+    addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     coffee-script (2.4.1)
       coffee-script-source

docs/configuration.md

@@ -722,7 +722,7 @@ Vimspector then orchestrates the various tools to set you up.
       "variables": {
         // Just an example of how to specify a variable manually rather than
         // vimspector asking for input from the user
-        "ServiceName": "${fileBasenameNoExtension}"
+        "ServiceName": "${fileBasenameNoExtention}"
       },
 
       "adapter": "python-remote",

python3/vimspector/code.py

@@ -16,6 +16,7 @@
 import vim
 import logging
 import json
+import os
 from collections import defaultdict
 
 from vimspector import utils, terminal, signs
@@ -40,6 +41,7 @@ class CodeView( object ):
       'breakpoints': []
     }
     self._current_frame = None
+    self._scratch_buffers = []
 
     with utils.LetCurrentWindow( self._window ):
       if utils.UseWinBar():
@@ -173,6 +175,10 @@ class CodeView( object ):
     self.ClearBreakpoints()
     self.Clear()
 
+    for b in self._scratch_buffers:
+      utils.CleanUpHiddenBuffer( b )
+    self._scratch_buffers = []
+
   def AddBreakpoints( self, source, breakpoints ):
     for breakpoint in breakpoints:
       source = breakpoint.get( 'source' ) or source
@@ -287,3 +293,32 @@ class CodeView( object ):
     # FIXME: Change this tor return the PID rather than having debug_session
     # work that out
     return self._terminal.buffer_number
+
+
+  def ShowMemory( self, memoryReference, length, offset, msg ):
+    if not self._window.valid:
+      return False
+
+    buf_name = os.path.join( '_vimspector_mem', memoryReference )
+    buf = utils.BufferForFile( buf_name )
+    self._scratch_buffers.append( buf )
+    utils.SetUpHiddenBuffer( buf, buf_name )
+    with utils.ModifiableScratchBuffer( buf ):
+      # TODO: The data is encoded in base64, so we need to convert that to the
+      # equivalent output of say xxd
+      data = msg.get( 'body', {} ).get( 'data', '' )
+      utils.SetBufferContents( buf, [
+        f'Memory Dump for Reference {memoryReference} Length: {length} bytes'
+        f' Offset: {offset}',
+        '-' * 80,
+        'Offset    Bytes                                             Text',
+        '-' * 80,
+      ] )
+      utils.AppendToBuffer( buf, utils.Base64ToHexDump( data ) )
+
+    utils.SetSyntax( '', 'xxd', buf )
+    utils.JumpToWindow( self._window )
+    utils.OpenFileInCurrentWindow( buf_name )
+
+    # TODO: Need to set up some mappings here that allow the user to browse
+    # around by setting the offset

python3/vimspector/debug_session.py

@@ -535,6 +535,45 @@ class DebugSession( object ):
   def SetVariableValue( self, new_value = None, buf = None, line_num = None ):
     self._variablesView.SetVariableValue( new_value, buf, line_num )
 
+  @IfConnected()
+  def ReadMemory( self, offset = None, buf = None, line_num = None ):
+    if not self._server_capabilities.get( 'supportsReadMemoryRequest' ):
+      utils.UserMessage( "Server does not support memory request",
+                         error = True )
+      return
+
+    memoryReference = self._variablesView.GetMemoryReference( buf, line_num )
+    if memoryReference is None:
+      utils.UserMessage( "Cannot find memory reference for that",
+                         error = True )
+      return
+
+    length = utils.AskForInput( 'How much data to display? ',
+                                default_value = '1024' )
+
+    if length is None:
+      return
+
+    offset = utils.AskForInput( 'Location offset? ',
+                                default_value = '0' )
+
+    if offset is None:
+      return
+
+
+    def handler( msg ):
+      self._codeView.ShowMemory( memoryReference, length, offset, msg )
+
+    self._connection.DoRequest( handler, {
+      'command': 'readMemory',
+      'arguments': {
+        'memoryReference': memoryReference,
+        'count': int( length ),
+        'offset': int( offset )
+      }
+    } )
+
+
   @IfConnected()
   def AddWatch( self, expression ):
     self._variablesView.AddWatch( self._stackTraceView.GetCurrentFrame(),
@@ -1163,7 +1202,8 @@ class DebugSession( object ):
         'pathFormat': 'path',
         'supportsVariableType': True,
         'supportsVariablePaging': False,
-        'supportsRunInTerminalRequest': True
+        'supportsRunInTerminalRequest': True,
+        'supportsMemoryReferences': True
       },
     } )
 

python3/vimspector/gadgets.py

@@ -30,12 +30,12 @@ GADGETS = {
                                                                 root,
                                                                 gadget ),
     'all': {
-      'version': '1.6.0',
+      'version': '0.27.0',
       "adapters": {
         "vscode-cpptools": {
           "name": "cppdbg",
           "command": [
-            "${gadgetDir}/vscode-cpptools/debugAdapters/bin/OpenDebugAD7"
+            "${gadgetDir}/vscode-cpptools/debugAdapters/OpenDebugAD7"
           ],
           "attach": {
             "pidProperty": "processId",
@@ -53,17 +53,17 @@ GADGETS = {
     'linux': {
       'file_name': 'cpptools-linux.vsix',
       'checksum':
-        'c25299bcfb46b22d41aa3f125df7184e6282a35ff9fb69c47def744cb4778f55',
+        '3695202e1e75a03de18049323b66d868165123f26151f8c974a480eaf0205435',
     },
     'macos': {
-      'file_name': 'cpptools-osx-arm64.vsix',
+      'file_name': 'cpptools-osx.vsix',
       'checksum':
-        'ceb3e8cdaa2b5bb45af50913ddd8402089969748af8d70f5d46480408287ba6f',
+        'cb061e3acd7559a539e5586f8d3f535101c4ec4e8a48195856d1d39380b5cf3c',
     },
     'windows': {
       'file_name': 'cpptools-win32.vsix',
       'checksum':
-        'ef7ac5831874a3c7dbf0feb826bfda2be579aff9b6d990622fff1d0d4ede00d1',
+        'aa294368ed16d48c59e49c8000e146eae5a19ad07b654efed5db8ec93b24229e',
       "adapters": {
         "vscode-cpptools": {
           "name": "cppdbg",
@@ -323,10 +323,10 @@ GADGETS = {
         '${version}/${file_name}',
     },
     'all': {
-            'version': 'v1.17.0',
-            'file_name': 'php-debug-1.17.0.vsix',
+      'version': 'v1.16.0',
+            'file_name': 'php-debug-1.16.0.vsix',
       'checksum':
-        'd0fff272503414b6696cc737bc2e18e060fdd5e5dc4bcaf38ae7373afd8d8bc9',
+        '62d210f7b87b21315c37ea10a1a5dbae376ff9f963b8f8cf33361e01413731be',
     },
     'adapters': {
       'vscode-php-debug': {
@@ -394,12 +394,12 @@ GADGETS = {
              '${version}/${file_name}',
     },
     'all': {
-      'version': 'v1.6.6',
+      'version': 'v1.6.4',
     },
     'macos': {
-      'file_name': 'codelldb-aarch64-darwin.vsix',
+      'file_name': 'codelldb-x86_64-darwin.vsix',
       'checksum':
-        '5adc3b9139eabdafd825bd5efc55df4424a203fb2b6087b425cd434956e7ec58',
+        'aa920b7b7d2ad4e9d70086355841b0b4844fb5f62cdea1296904100a1b660776',
       'make_executable': [
         'adapter/codelldb',
         'lldb/bin/debugserver',
@@ -410,7 +410,7 @@ GADGETS = {
     'linux': {
       'file_name': 'codelldb-x86_64-linux.vsix',
       'checksum':
-        'eda2cd9b3089dcc0524c273e91ffb5875fe08c930bf643739a2cd1846e1f98d6',
+        '',
       'make_executable': [
         'adapter/codelldb',
         'lldb/bin/lldb',
@@ -421,7 +421,7 @@ GADGETS = {
     'windows': {
       'file_name': 'codelldb-x86_64-windows.vsix',
       'checksum':
-        '8ddebe8381a3d22dc3d95139c3797fda06b5cc34aadf300e13b1c516b9da95fe',
+        '',
       'make_executable': []
     },
     'adapters': {

python3/vimspector/installer.py

@@ -358,8 +358,7 @@ def InstallCppTools( name, root, gadget ):
   # It's hilarious, but the execute bits aren't set in the vsix. So they
   # actually have javascript code which does this. It's just a horrible horrible
   # hack that really is not funny.
-  MakeExecutable(
-    os.path.join( extension, 'debugAdapters', 'bin', 'OpenDebugAD7' ) )
+  MakeExecutable( os.path.join( extension, 'debugAdapters', 'OpenDebugAD7' ) )
   with open( os.path.join( extension, 'package.json' ) ) as f:
     package = json.load( f )
     runtime_dependencies = package[ 'runtimeDependencies' ]

python3/vimspector/output.py

@@ -32,7 +32,6 @@ class TabBuffer( object ):
 BUFFER_MAP = {
   'console': 'Console',
   'stdout': 'Console',
-  'output': 'Console',
   'stderr': 'stderr',
   'telemetry': None,
 }

python3/vimspector/settings.py

@@ -54,7 +54,8 @@ DEFAULTS = {
     'variables': {
       'expand_collapse': [ '<CR>', '<2-LeftMouse>' ],
       'delete': [ '<Del>' ],
-      'set_value': [ '<C-CR>', '<leader><CR>' ]
+      'set_value': [ '<C-CR>', '<leader><CR>' ],
+      'read_memory': [ '<leader>m' ],
     },
     'stack_trace': {
       'expand_or_jump': [ '<CR>', '<2-LeftMouse>' ],

python3/vimspector/utils.py

@@ -25,7 +25,9 @@ import shlex
 import collections
 import re
 import typing
+import base64
 
+from vimspector.vendor.hexdump import hexdump
 
 LOG_FILE = os.path.expanduser( os.path.join( '~', '.vimspector.log' ) )
 
@@ -864,3 +866,8 @@ def UseWinBar():
   # Buggy neovim doesn't render correctly when the WinBar is defined:
   # https://github.com/neovim/neovim/issues/12689
   return not int( Call( 'has', 'nvim' ) )
+
+
+def Base64ToHexDump( data ):
+  data = base64.b64decode( data )
+  return list( hexdump( data, 'generator' ) )

python3/vimspector/variables.py

@@ -56,6 +56,11 @@ class Expandable:
   def VariablesReference( self ):
     assert False
 
+  @abc.abstractmethod
+  def MemoryReference( self ):
+    assert None
+
+
 
 class Scope( Expandable ):
   """Holds an expandable scope (a DAP scope dict), with expand/collapse state"""
@@ -66,6 +71,9 @@ class Scope( Expandable ):
   def VariablesReference( self ):
     return self.scope.get( 'variablesReference', 0 )
 
+  def MemoryReference( self ):
+    return None
+
   def Update( self, scope ):
     self.scope = scope
 
@@ -81,6 +89,9 @@ class WatchResult( Expandable ):
   def VariablesReference( self ):
     return self.result.get( 'variablesReference', 0 )
 
+  def MemoryReference( self ):
+    return self.result.get( 'memoryReference' )
+
   def Update( self, result ):
     self.changed = False
     if self.result[ 'result' ] != result[ 'result' ]:
@@ -105,6 +116,9 @@ class Variable( Expandable ):
   def VariablesReference( self ):
     return self.variable.get( 'variablesReference', 0 )
 
+  def MemoryReference( self ):
+    return self.variable.get( 'memoryReference' )
+
   def Update( self, variable ):
     self.changed = False
     if self.variable[ 'value' ] != variable[ 'value' ]:
@@ -163,6 +177,10 @@ def AddExpandMappings( mappings = None ):
   for mapping in utils.GetVimList( mappings, 'set_value' ):
     vim.command( f'nnoremap <silent> <buffer> { mapping } '
                  ':<C-u>call vimspector#SetVariableValue()<CR>' )
+  for mapping in utils.GetVimList( mappings, 'read_memory' ):
+    vim.command( f'nnoremap <silent> <buffer> { mapping } '
+                 ':<C-u>call vimspector#ReadMemory()<CR>' )
+
 
 
 class VariablesView( object ):
@@ -187,6 +205,8 @@ class VariablesView( object ):
       if utils.UseWinBar():
         vim.command( 'nnoremenu <silent> 1.1 WinBar.Set '
                      ':call vimspector#SetVariableValue()<CR>' )
+        vim.command( 'nnoremenu <silent> 1.2 WinBar.Memory '
+                     ':call vimspector#ReadMemory()<CR>' )
       AddExpandMappings( mappings )
 
     # Set up the "Watches" buffer in the watches_win (and create a WinBar in
@@ -211,8 +231,10 @@ class VariablesView( object ):
                      ':call vimspector#ExpandVariable()<CR>' )
         vim.command( 'nnoremenu <silent> 1.3 WinBar.Delete '
                      ':call vimspector#DeleteWatch()<CR>' )
-        vim.command( 'nnoremenu <silent> 1.1 WinBar.Set '
+        vim.command( 'nnoremenu <silent> 1.4 WinBar.Set '
                      ':call vimspector#SetVariableValue()<CR>' )
+        vim.command( 'nnoremenu <silent> 1.5 WinBar.Memory '
+                     ':call vimspector#ReadMemory()<CR>' )
 
     # Set the (global!) balloon expr if supported
     has_balloon      = int( vim.eval( "has( 'balloon_eval' )" ) )
@@ -580,6 +602,14 @@ class VariablesView( object ):
     }, failure_handler = failure_handler )
 
 
+  def GetMemoryReference( self, buf = None, line_num = None ):
+    # Get a memoryReference for use in a ReadMemory request
+    variable, _ = self._GetVariable( buf, line_num )
+    if variable is None:
+      return None
+
+    return variable.MemoryReference()
+
 
   def _DrawVariables( self, view, variables, indent, is_short = False ):
     assert indent > 0
@@ -595,10 +625,12 @@ class VariablesView( object ):
           value = variable.variable.get( 'value', '<unknown>' )
         )
       else:
+        marker = 'm' if variable.MemoryReference() is not None else ' '
+        marker += '*' if variable.changed else ' '
         text = '{indent}{marker}{icon} {name} ({type_}): {value}'.format(
           # We borrow 1 space of indent to draw the change marker
           indent = ' ' * ( indent - 1 ),
-          marker = '*' if variable.changed else ' ',
+          marker = marker,
           icon = '+' if ( variable.IsExpandable()
                           and not variable.IsExpanded() ) else '-',
           name = variable.variable.get( 'name', '' ),

python3/vimspector/vendor/hexdump.py

@@ -0,0 +1,466 @@
+#!/usr/bin/env python
+# -*- coding: latin-1 -*-
+
+# <-- removing this magic comment breaks Python 3.4 on Windows
+"""
+1. Dump binary data to the following text format:
+
+00000000: 00 00 00 5B 68 65 78 64  75 6D 70 5D 00 00 00 00  ...[hexdump]....
+00000010: 00 11 22 33 44 55 66 77  88 99 AA BB CC DD EE FF  .."3DUfw........
+
+It is similar to the one used by:
+Scapy
+00 00 00 5B 68 65 78 64 75 6D 70 5D 00 00 00 00  ...[hexdump]....
+00 11 22 33 44 55 66 77 88 99 AA BB CC DD EE FF  .."3DUfw........
+
+Far Manager
+000000000: 00 00 00 5B 68 65 78 64 ¦ 75 6D 70 5D 00 00 00 00     [hexdump]
+000000010: 00 11 22 33 44 55 66 77 ¦ 88 99 AA BB CC DD EE FF   ?"3DUfwˆ™ª»ÌÝîÿ
+
+
+2. Restore binary data from the formats above as well
+   as from less exotic strings of raw hex
+
+"""
+
+__version__ = '3.3'
+__author__  = 'anatoly techtonik <techtonik@gmail.com>'
+__license__ = 'Public Domain'
+
+__history__ = \
+"""
+3.3 (2015-01-22)
+ * accept input from sys.stdin if "-" is specified
+   for both dump and restore (issue #1)
+ * new normalize_py() helper to set sys.stdout to
+   binary mode on Windows
+
+3.2 (2015-07-02)
+ * hexdump is now packaged as .zip on all platforms
+   (on Linux created archive was tar.gz)
+ * .zip is executable! try `python hexdump-3.2.zip`
+ * dump() now accepts configurable separator, patch
+   by Ian Land (PR #3)
+
+3.1 (2014-10-20)
+ * implemented workaround against mysterious coding
+   issue with Python 3 (see revision 51302cf)
+ * fix Python 3 installs for systems where UTF-8 is
+   not default (Windows), thanks to George Schizas
+   (the problem was caused by reading of README.txt)
+
+3.0 (2014-09-07)
+ * remove unused int2byte() helper
+ * add dehex(text) helper to convert hex string
+   to binary data
+ * add 'size' argument to dump() helper to specify
+   length of chunks
+
+2.0 (2014-02-02)
+ * add --restore option to command line mode to get
+   binary data back from hex dump
+ * support saving test output with `--test logfile`
+ * restore() from hex strings without spaces
+ * restore() now raises TypeError if input data is
+   not string
+ * hexdump() and dumpgen() now don't return unicode
+   strings in Python 2.x when generator is requested
+
+1.0 (2013-12-30)
+ * length of address is reduced from 10 to 8
+ * hexdump() got new 'result' keyword argument, it
+   can be either 'print', 'generator' or 'return'
+ * actual dumping logic is now in new dumpgen()
+   generator function
+ * new dump(binary) function that takes binary data
+   and returns string like "66 6F 72 6D 61 74"
+ * new genchunks(mixed, size) function that chunks
+   both sequences and file like objects
+
+0.5 (2013-06-10)
+ * hexdump is now also a command line utility (no
+   restore yet)
+
+0.4 (2013-06-09)
+ * fix installation with Python 3 for non English
+   versions of Windows, thanks to George Schizas
+
+0.3 (2013-04-29)
+ * fully Python 3 compatible
+
+0.2 (2013-04-28)
+ * restore() to recover binary data from a hex dump in
+   native, Far Manager and Scapy text formats (others
+   might work as well)
+ * restore() is Python 3 compatible
+
+0.1 (2013-04-28)
+ * working hexdump() function for Python 2
+"""
+
+import binascii  # binascii is required for Python 3
+import sys
+
+# --- constants
+PY3K = sys.version_info >= (3, 0)
+
+# --- workaround against Python consistency issues
+def normalize_py():
+  ''' Problem 001 - sys.stdout in Python is by default opened in
+      text mode, and writes to this stdout produce corrupted binary
+      data on Windows
+
+          python -c "import sys; sys.stdout.write('_\n_')" > file
+          python -c "print(repr(open('file', 'rb').read()))"
+  '''
+  if sys.platform == "win32":
+    # set sys.stdout to binary mode on Windows
+    import os, msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+
+# --- - chunking helpers
+def chunks(seq, size):
+  '''Generator that cuts sequence (bytes, memoryview, etc.)
+     into chunks of given size. If `seq` length is not multiply
+     of `size`, the lengh of the last chunk returned will be
+     less than requested.
+
+     >>> list( chunks([1,2,3,4,5,6,7], 3) )
+     [[1, 2, 3], [4, 5, 6], [7]]
+  '''
+  d, m = divmod(len(seq), size)
+  for i in range(d):
+    yield seq[i*size:(i+1)*size]
+  if m:
+    yield seq[d*size:]
+
+def chunkread(f, size):
+  '''Generator that reads from file like object. May return less
+     data than requested on the last read.'''
+  c = f.read(size)
+  while len(c):
+    yield c
+    c = f.read(size)
+
+def genchunks(mixed, size):
+  '''Generator to chunk binary sequences or file like objects.
+     The size of the last chunk returned may be less than
+     requested.'''
+  if hasattr(mixed, 'read'):
+    return chunkread(mixed, size)
+  else:
+    return chunks(mixed, size)
+# --- - /chunking helpers
+
+
+def dehex(hextext):
+  """
+  Convert from hex string to binary data stripping
+  whitespaces from `hextext` if necessary.
+  """
+  if PY3K:
+    return bytes.fromhex(hextext)
+  else:
+    hextext = "".join(hextext.split())
+    return hextext.decode('hex')
+
+def dump(binary, size=2, sep=' '):
+  '''
+  Convert binary data (bytes in Python 3 and str in
+  Python 2) to hex string like '00 DE AD BE EF'.
+  `size` argument specifies length of text chunks
+  and `sep` sets chunk separator.
+  '''
+  hexstr = binascii.hexlify(binary)
+  if PY3K:
+    hexstr = hexstr.decode('ascii')
+  return sep.join(chunks(hexstr.upper(), size))
+
+def dumpgen(data):
+  '''
+  Generator that produces strings:
+
+  '00000000: 00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................'
+  '''
+  generator = genchunks(data, 16)
+  for addr, d in enumerate(generator):
+    # 00000000:
+    line = '%08X: ' % (addr*16)
+    # 00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00 
+    dumpstr = dump(d)
+    line += dumpstr[:8*3]
+    if len(d) > 8:  # insert separator if needed
+      line += ' ' + dumpstr[8*3:]
+    # ................
+    # calculate indentation, which may be different for the last line
+    pad = 2
+    if len(d) < 16:
+      pad += 3*(16 - len(d))
+    if len(d) <= 8:
+      pad += 1
+    line += ' '*pad
+
+    for byte in d:
+      # printable ASCII range 0x20 to 0x7E
+      if not PY3K:
+        byte = ord(byte)
+      if 0x20 <= byte <= 0x7E:
+        line += chr(byte)
+      else:
+        line += '.'
+    yield line
+  
+def hexdump(data, result='print'):
+  '''
+  Transform binary data to the hex dump text format:
+
+  00000000: 00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  ................
+
+    [x] data argument as a binary string
+    [x] data argument as a file like object
+
+  Returns result depending on the `result` argument:
+    'print'     - prints line by line
+    'return'    - returns single string
+    'generator' - returns generator that produces lines
+  '''
+  if PY3K and type(data) == str:
+    raise TypeError('Abstract unicode data (expected bytes sequence)')
+
+  gen = dumpgen(data)
+  if result == 'generator':
+    return gen
+  elif result == 'return':
+    return '\n'.join(gen)
+  elif result == 'print':
+    for line in gen:
+      print(line)
+  else:
+    raise ValueError('Unknown value of `result` argument')
+
+def restore(dump):
+  '''
+  Restore binary data from a hex dump.
+    [x] dump argument as a string
+    [ ] dump argument as a line iterator
+
+  Supported formats:
+    [x] hexdump.hexdump
+    [x] Scapy
+    [x] Far Manager
+  '''
+  minhexwidth = 2*16    # minimal width of the hex part - 00000... style
+  bytehexwidth = 3*16-1 # min width for a bytewise dump - 00 00 ... style
+
+  result = bytes() if PY3K else ''
+  if type(dump) != str:
+    raise TypeError('Invalid data for restore')
+
+  text = dump.strip()  # ignore surrounding empty lines
+  for line in text.split('\n'):
+    # strip address part
+    addrend = line.find(':')
+    if 0 < addrend < minhexwidth:  # : is not in ascii part
+      line = line[addrend+1:]
+    line = line.lstrip()
+    # check dump type
+    if line[2] == ' ':  # 00 00 00 ...  type of dump
+      # check separator
+      sepstart = (2+1)*7+2  # ('00'+' ')*7+'00'
+      sep = line[sepstart:sepstart+3]
+      if sep[:2] == '  ' and sep[2:] != ' ':  # ...00 00  00 00...
+        hexdata = line[:bytehexwidth+1]
+      elif sep[2:] == ' ':  # ...00 00 | 00 00...  - Far Manager
+        hexdata = line[:sepstart] + line[sepstart+3:bytehexwidth+2]
+      else:                 # ...00 00 00 00... - Scapy, no separator
+        hexdata = line[:bytehexwidth]
+      line = hexdata
+    result += dehex(line)
+  return result
+
+
+def runtest(logfile=None):
+  '''Run hexdump tests. Requires hexfile.bin to be in the same
+     directory as hexdump.py itself'''
+
+  class TeeOutput(object):
+    def __init__(self, stream1, stream2):
+      self.outputs = [stream1, stream2]
+
+    # -- methods from sys.stdout / sys.stderr
+    def write(self, data):
+      for stream in self.outputs:
+        if PY3K:
+          if 'b' in stream.mode:
+            data = data.encode('utf-8')
+        stream.write(data)
+        stream.flush()
+
+    def tell(self):
+      raise IOError
+
+    def flush(self):
+      for stream in self.outputs:
+        stream.flush()
+    # --/ sys.stdout
+
+  if logfile:
+    openlog = open(logfile, 'wb')
+    # copy stdout and stderr streams to log file
+    savedstd = sys.stderr, sys.stdout
+    sys.stderr = TeeOutput(sys.stderr, openlog)
+    sys.stdout = TeeOutput(sys.stdout, openlog)
+    
+
+  def echo(msg, linefeed=True):
+    sys.stdout.write(msg)
+    if linefeed:
+      sys.stdout.write('\n')
+
+  expected = '''\
+00000000: 00 00 00 5B 68 65 78 64  75 6D 70 5D 00 00 00 00  ...[hexdump]....
+00000010: 00 11 22 33 44 55 66 77  88 99 0A BB CC DD EE FF  .."3DUfw........\
+'''
+
+  # get path to hexfile.bin
+  # this doesn't work from .zip
+  #   import os.path as osp
+  #   hexfile = osp.dirname(osp.abspath(__file__)) + '/hexfile.bin'
+  # this doesn't work either
+  #   hexfile = osp.dirname(sys.modules[__name__].__file__) + '/hexfile.bin'
+  # this works
+  import pkgutil
+  bin = pkgutil.get_data('hexdump', 'data/hexfile.bin')
+
+  # varios length of input data
+  hexdump(b'zzzz'*12)
+  hexdump(b'o'*17)
+  hexdump(b'p'*24)
+  hexdump(b'q'*26)
+  # allowable character set filter
+  hexdump(b'line\nfeed\r\ntest')
+  hexdump(b'\x00\x00\x00\x5B\x68\x65\x78\x64\x75\x6D\x70\x5D\x00\x00\x00\x00'
+          b'\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\x0A\xBB\xCC\xDD\xEE\xFF')
+  print('---')
+  # dumping file-like binary object to screen (default behavior)
+  hexdump(bin)
+  print('return output')
+  hexout = hexdump(bin, result='return')
+  assert hexout == expected, 'returned hex didn\'t match'
+  print('return generator')
+  hexgen = hexdump(bin, result='generator')
+  assert next(hexgen) == expected.split('\n')[0], 'hex generator 1 didn\'t match'
+  assert next(hexgen) == expected.split('\n')[1], 'hex generator 2 didn\'t match'
+
+  # binary restore test
+  bindata = restore(
+'''
+00000000: 00 00 00 5B 68 65 78 64  75 6D 70 5D 00 00 00 00  ...[hexdump]....
+00000010: 00 11 22 33 44 55 66 77  88 99 0A BB CC DD EE FF  .."3DUfw........
+''')
+  echo('restore check ', linefeed=False)
+  assert bin == bindata, 'restore check failed'
+  echo('passed')
+
+  far = \
+'''
+000000000: 00 00 00 5B 68 65 78 64 ¦ 75 6D 70 5D 00 00 00 00     [hexdump]
+000000010: 00 11 22 33 44 55 66 77 ¦ 88 99 0A BB CC DD EE FF   ?"3DUfwˆ™ª»ÌÝîÿ
+'''
+  echo('restore far format ', linefeed=False)
+  assert bin == restore(far), 'far format check failed'
+  echo('passed')
+
+  scapy = '''\
+00 00 00 5B 68 65 78 64 75 6D 70 5D 00 00 00 00  ...[hexdump]....
+00 11 22 33 44 55 66 77 88 99 0A BB CC DD EE FF  .."3DUfw........
+'''
+  echo('restore scapy format ', linefeed=False)
+  assert bin == restore(scapy), 'scapy format check failed'
+  echo('passed')
+
+  if not PY3K:
+    assert restore('5B68657864756D705D') == '[hexdump]', 'no space check failed'
+    assert dump('\\\xa1\xab\x1e', sep='').lower() == '5ca1ab1e'
+  else:
+    assert restore('5B68657864756D705D') == b'[hexdump]', 'no space check failed'
+    assert dump(b'\\\xa1\xab\x1e', sep='').lower() == '5ca1ab1e'
+
+  print('---[test file hexdumping]---')
+
+  import os
+  import tempfile
+  hexfile = tempfile.NamedTemporaryFile(delete=False)
+  try:
+    hexfile.write(bin)
+    hexfile.close()
+    hexdump(open(hexfile.name, 'rb'))
+  finally:
+    os.remove(hexfile.name)
+  if logfile:
+    sys.stderr, sys.stdout = savedstd
+    openlog.close()
+
+
+def main():
+  from optparse import OptionParser
+  parser = OptionParser(usage='''
+  %prog [binfile|-]
+  %prog -r hexfile
+  %prog --test [logfile]''', version=__version__)
+  parser.add_option('-r', '--restore', action='store_true',
+                                       help='restore binary from hex dump')
+  parser.add_option('--test', action='store_true', help='run hexdump sanity checks')
+
+  options, args = parser.parse_args()
+
+  if options.test:
+    if args:
+      runtest(logfile=args[0])
+    else:
+      runtest()
+  elif not args or len(args) > 1:
+    parser.print_help()
+    sys.exit(-1)
+  else:
+    ## dump file
+    if not options.restore:
+      # [x] memory effective dump
+      if args[0] == '-':
+        if not PY3K:
+          hexdump(sys.stdin)
+        else:
+          hexdump(sys.stdin.buffer)
+      else:
+        hexdump(open(args[0], 'rb'))
+
+    ## restore file
+    else:
+      # prepare input stream
+      if args[0] == '-':
+        instream = sys.stdin
+      else:
+        if PY3K:
+          instream = open(args[0])
+        else:
+          instream = open(args[0], 'rb')
+
+      # output stream
+      # [ ] memory efficient restore
+      if PY3K:
+        sys.stdout.buffer.write(restore(instream.read()))
+      else:
+        # Windows - binary mode for sys.stdout to prevent data corruption
+        normalize_py()
+        sys.stdout.write(restore(instream.read()))
+
+if __name__ == '__main__':
+  main()
+
+# [x] file restore from command line utility
+# [ ] write dump with LF on Windows for consistency
+# [ ] encoding param for hexdump()ing Python 3 str if anybody requests that
+
+# [ ] document chunking API
+# [ ] document hexdump API
+# [ ] blog about sys.stdout text mode problem on Windows

run_tests

@@ -21,7 +21,7 @@ out_fd=1
 
 while [ -n "$1" ]; do
   case "$1" in
-    "--basedir"|"--base-dir"|"--test-base")
+    "--basedir")
       shift
       SetBaseDir $1
       shift
@@ -36,7 +36,7 @@ while [ -n "$1" ]; do
       INSTALL=$1
       shift
       ;;
-    "--update"|"--upgrade")
+    "--update")
       UPDATE=1
       shift
       ;;

support/custom_ui_vimrc

@@ -54,7 +54,7 @@ function s:SetUpTerminal()
   let padding = 4
   let cols = max( [ min( [ &columns - left_bar - code - padding, 80 ] ), 10 ] )
   call win_gotoid( terminal_win )
-  execute string(cols) . 'wincmd |'
+  execute cols . 'wincmd |'
 endfunction
 
 function! s:CustomiseWinBar()

support/gadget_upgrade/README.md

@@ -1,8 +0,0 @@
-# Manually updating shipped gadgets
-
-Download the gadget files manuall from their official source into this dir.
-Run `./checksum.py <list of files>` to get the checksums.
-
-Update ../../python3/vimspector/gadgets.py with the new version and the
-checksums.
-

support/gadget_upgrade/checksum.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-
-import hashlib
-import sys
-
-
-def GetChecksumSHA254( file_path ):
-  with open( file_path, 'rb' ) as existing_file:
-    return hashlib.sha256( existing_file.read() ).hexdigest()
-
-
-for arg in sys.argv[ 1: ]:
-  print( f"{ arg } = { GetChecksumSHA254( arg ) }" )

support/test/bash/.vimspector.json

@@ -1,15 +0,0 @@
-{
-  "$schema": "https://puremourning.github.io/vimspector/schema/vimspector.schema.json",
-  "configurations": {
-    "Run Current Script": {
-      "adapter": "vscode-bash",
-      "autoselect": false,
-      "configuration": {
-        "request": "launch",
-        "program": "${file}",
-        "cwd": "${fileDirname}",
-        "args": [ "*${args}" ]
-      }
-    }
-  }
-}

support/test/go/name-starts-with-vowel/.vimspector.json

@@ -1,29 +0,0 @@
-{
-    "configurations": {
-        "run-cmd": {
-            "adapter": "vscode-go",
-            "configuration": {
-                "request": "launch",
-                "program": "${workspaceRoot}/cmd/namestartswithvowel/main.go",
-                "mode": "debug",
-                "dlvToolPath": "$HOME/go/bin/dlv",
-                "dlvLoadConfig": {
-                    "maxArrayValues": 1000,
-                    "maxStringLen": 1000
-                }
-            }
-        },
-        "test-current-file": {
-            "adapter": "vscode-go",
-            "configuration": {
-                "request": "launch",
-                "mode": "test",
-                "program": "${fileDirname}",
-                "cwd": "${fileDirname}",
-                "dlvToolPath": "$GOPATH/bin/dlv",
-                "env": {},
-                "args": []
-            }
-        }
-    }
-}

support/test/go/name-starts-with-vowel/README.md

@@ -1,33 +0,0 @@
-# Purpose
-
-This example comes with two example vimspector configs for the Go programming language.
-
-1) `run-cmd` will launch the main programme under `cmd/namestartswithvowel`.
-1) `test-current-file` will run the tests in the current file in debug mode.
-
-## Example use-cases
-
-### run-cmd
-
-* Open `cmd/namestartswithvowel/main.go`
-* Add a breakpoint somewhere within the programme
-* Start the debugger (`:call vimspector#Continue()` or your relevant keymapping)
-* Select the first launch configuration (`1: run-cmd`)
-
-### test-current-file
-
-* Open `internal/vowels/vowels_test.go`
-* Add a breakpoint somewhere within the test
-* Start the debugger (`:call vimspector#Continue()` or your relevant keymapping)
-* Select the second launch configuration (`2: test-current-file`)
-
-## Additional Configuration
-
-There are two additional configuration options specified under `run-cmd`; these parameters configure the maximum string/array size to be shown while debugging.
-
-```
-"dlvLoadConfig": {
-    "maxArrayValues": 1000,
-    "maxStringLen": 1000
-}
-```

support/test/go/name-starts-with-vowel/cmd/namestartswithvowel/main.go

@@ -1,20 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"example.com/internal/vowels"
-)
-
-func main() {
-	names := []string{"Simon", "Bob", "Jennifer", "Amy", "Duke", "Elizabeth"}
-
-	for _, n := range names {
-		if vowels.NameStartsWithVowel(n) {
-			fmt.Printf("%s starts with a vowel!\n", n)
-			continue
-		}
-
-		fmt.Printf("%s does not start with a vowel!\n", n)
-	}
-}

support/test/go/name-starts-with-vowel/go.mod

@@ -1,3 +0,0 @@
-module example.com
-
-go 1.16

support/test/go/name-starts-with-vowel/internal/vowels/vowels.go

@@ -1,9 +0,0 @@
-package vowels
-
-import "strings"
-
-func NameStartsWithVowel(name string) bool {
-	s := strings.Split(strings.ToLower(name), "")
-
-	return s[0] == "a" || s[0] == "e" || s[0] == "i" || s[0] == "o" || s[0] == "u"
-}

support/test/go/name-starts-with-vowel/internal/vowels/vowels_test.go

@@ -1,30 +0,0 @@
-package vowels
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestNameStartsWithVowel(t *testing.T) {
-	testCases := []struct {
-		input          string
-		expectedOutput bool
-	}{
-		{
-			input:          "Simon",
-			expectedOutput: false,
-		},
-		{
-			input:          "Andy",
-			expectedOutput: true,
-		},
-	}
-	for _, tt := range testCases {
-		t.Run(fmt.Sprintf("%s should product %t", tt.input, tt.expectedOutput), func(t *testing.T) {
-			out := NameStartsWithVowel(tt.input)
-			if out != tt.expectedOutput {
-				t.Errorf("%s produced %t, when %t was expected", tt.input, out, tt.expectedOutput)
-			}
-		})
-	}
-}

tests/testdata/cpp/simple/.vimspector.json

@@ -12,7 +12,7 @@
         "externalConsole": false,
         "stopAtEntry": true,
         "stopOnEntry": true,
-        "MIMode": "${VIMSPECTOR_MIMODE}"
+        "MImode": "${VIMSPECTOR_MIMODE}"
       },
       "breakpoints": {
         "exception": {
@@ -33,7 +33,7 @@
         "externalConsole": false,
         "stopAtEntry": false,
         "stopOnEntry": false,
-        "MIMode": "${VIMSPECTOR_MIMODE}"
+        "MImode": "${VIMSPECTOR_MIMODE}"
       },
       "breakpoints": {
         "exception": {
@@ -55,7 +55,7 @@
         "externalConsole": false,
         "stopAtEntry": false,
         "stopOnEntry": false,
-        "MIMode": "${VIMSPECTOR_MIMODE}"
+        "MImode": "${VIMSPECTOR_MIMODE}"
       },
       "breakpoints": {
         "exception": {
@@ -82,7 +82,7 @@
       "configuration": {
         "request": "launch",
         "program": "${workspaceRoot}/${fileBasenameNoExtension}",
-        "MIMode": "${VIMSPECTOR_MIMODE}",
+        "MImode": "${VIMSPECTOR_MIMODE}",
         "externalConsole": false,
         "args": [
           "CALCULATED_LIST", "${CALCULATED_LIST}",

tests/variables.test.vim

@@ -194,7 +194,6 @@ function! Test_ExpandVariables()
         \     [
         \       '- Scope: Locals',
         \       ' *+ t (Test): {...}',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -220,7 +219,6 @@ function! Test_ExpandVariables()
         \       '   \*- c (char): 0 ''\\0\{1,3}''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -239,7 +237,6 @@ function! Test_ExpandVariables()
         \       '    - c (char): 0 ''\\0\{1,3}''',
         \       '    - fffff (float): 0',
         \       '    + another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -256,7 +253,6 @@ function! Test_ExpandVariables()
         \     [
         \       '- Scope: Locals',
         \       '  + t (Test): {...}',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -271,7 +267,6 @@ function! Test_ExpandVariables()
         \     [
         \       '- Scope: Locals',
         \       '  + t (Test): {...}',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -291,7 +286,6 @@ function! Test_ExpandVariables()
         \       '   \*- c (char): 99 ''c''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -308,7 +302,6 @@ function! Test_ExpandVariables()
         \   assert_equal(
         \     [
         \       '+ Scope: Locals',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -323,7 +316,6 @@ function! Test_ExpandVariables()
         \   assert_equal(
         \     [
         \       '+ Scope: Locals',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -339,7 +331,6 @@ function! Test_ExpandVariables()
         \   assert_equal(
         \     [
         \       '+ Scope: Locals',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -855,7 +846,6 @@ function! Test_SetVariableValue_Local()
         \     [
         \       '- Scope: Locals',
         \       ' *+ t (Test): {...}',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -881,7 +871,6 @@ function! Test_SetVariableValue_Local()
         \       '   \*- c (char): 0 ''\\0\{1,3}''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -908,7 +897,6 @@ EOF
         \       '   \*- c (char): 0 ''\\0\{1,3}''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -928,7 +916,6 @@ EOF
         \       '   \*- c (char): 0 ''\\0\{1,3}''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,
@@ -948,7 +935,6 @@ EOF
         \       '   \*- c (char): 0 ''\\0\{1,3}''',
         \       '   \*- fffff (float): 0',
         \       '   \*+ another_test (AnotherTest):\( {...}\)\?',
-        \       '+ Scope: Registers',
         \     ],
         \     getbufline( winbufnr( g:vimspector_session_windows.variables ),
         \                 1,