Move installation routines into proper library

install_gadget.py

@@ -21,23 +21,13 @@ if sys.version_info.major < 3:
   sys.exit( "You need to run this with python 3. Your version is " +
             '.'.join( map( str, sys.version_info[ :3 ] ) ) )
 
-from urllib import request
 import argparse
-import contextlib
 import os
 import string
-import zipfile
-import gzip
-import shutil
 import subprocess
 import traceback
-import tarfile
-import hashlib
 import json
 import functools
-import time
-import ssl
-import io
 import operator
 import glob
 
@@ -45,7 +35,7 @@ import glob
 sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ),
                                   'python3' ) )
 
-from vimspector import install
+from vimspector import install, installer
 
 GADGETS = {
   'vscode-cpptools': {
@@ -210,7 +200,7 @@ GADGETS = {
       'file_name': 'netcoredbg-linux-master.tar.gz',
       'checksum': '',
     },
-    'do': lambda name, root, gadget: MakeSymlink(
+    'do': lambda name, root, gadget: installer.MakeSymlink(
       gadget_dir,
       name,
       os.path.join( root, 'netcoredbg' ) ),
@@ -393,29 +383,15 @@ GADGETS = {
 }
 
 
-@contextlib.contextmanager
-def CurrentWorkingDir( d ):
-  cur_d = os.getcwd()
-  try:
-    os.chdir( d )
-    yield
-  finally:
-    os.chdir( cur_d )
-
-
-def MakeExecutable( file_path ):
-  # TODO: import stat and use them by _just_ adding the X bit.
-  print( 'Making executable: {}'.format( file_path ) )
-  os.chmod( file_path, 0o755 )
-
-
 def InstallCppTools( name, root, gadget ):
   extension = os.path.join( root, 'extension' )
 
   # It's hilarious, but the execute bits aren't set in the vsix. So they
   # actually have javascript code which does this. It's just a horrible horrible
   # hack that really is not funny.
-  MakeExecutable( os.path.join( extension, 'debugAdapters', 'OpenDebugAD7' ) )
+  installer.MakeExecutable( os.path.join( extension,
+                                          'debugAdapters',
+                                          'OpenDebugAD7' ) )
   with open( os.path.join( extension, 'package.json' ) ) as f:
     package = json.load( f )
     runtime_dependencies = package[ 'runtimeDependencies' ]
@@ -423,14 +399,17 @@ def InstallCppTools( name, root, gadget ):
       for binary in dependency.get( 'binaries' ):
         file_path = os.path.abspath( os.path.join( extension, binary ) )
         if os.path.exists( file_path ):
-          MakeExecutable( os.path.join( extension, binary ) )
+          installer.MakeExecutable( os.path.join( extension, binary ) )
 
-  MakeExtensionSymlink( name, root )
+  installer.MakeExtensionSymlink( vimspector_base, name, root )
 
 
 def InstallBashDebug( name, root, gadget ):
-  MakeExecutable( os.path.join( root, 'extension', 'bashdb_dir', 'bashdb' ) )
+  installer.MakeExecutable( os.path.join( root,
-  MakeExtensionSymlink( name, root )
+                                          'extension',
+                                          'bashdb_dir',
+                                          'bashdb' ) )
+  installer.MakeExtensionSymlink( vimspector_base, name, root )
 
 
 def InstallDebugpy( name, root, gadget ):
@@ -442,7 +421,7 @@ def InstallDebugpy( name, root, gadget ):
   finally:
     os.chdir( wd )
 
-  MakeSymlink( gadget_dir, name, root )
+  installer.MakeSymlink( gadget_dir, name, root )
 
 
 def InstallTclProDebug( name, root, gadget ):
@@ -473,11 +452,11 @@ def InstallTclProDebug( name, root, gadget ):
         break
 
 
-  with CurrentWorkingDir( os.path.join( root, 'lib', 'tclparser' ) ):
+  with installer.CurrentWorkingDir( os.path.join( root, 'lib', 'tclparser' ) ):
     subprocess.check_call( configure )
     subprocess.check_call( [ 'make' ] )
 
-  MakeSymlink( gadget_dir, name, root )
+  installer.MakeSymlink( gadget_dir, name, root )
 
 
 def InstallNodeDebug( name, root, gadget ):
@@ -495,176 +474,10 @@ def InstallNodeDebug( name, root, gadget ):
     print( "  $ ./install_gadget.py --enable-node ..." )
     raise RuntimeError( 'Invalid node environent for node debugger' )
 
-  with CurrentWorkingDir( root ):
+  with installer.CurrentWorkingDir( root ):
     subprocess.check_call( [ 'npm', 'install' ] )
     subprocess.check_call( [ 'npm', 'run', 'build' ] )
-  MakeSymlink( gadget_dir, name, root )
+  installer.MakeSymlink( gadget_dir, name, root )
-
-
-def WithRetry( f ):
-  retries = 5
-  timeout = 1 # seconds
-
-  @functools.wraps( f )
-  def wrapper( *args, **kwargs ):
-    thrown = None
-    for _ in range( retries ):
-      try:
-        return f( *args, **kwargs )
-      except Exception as e:
-        thrown = e
-        print( "Failed - {}, will retry in {} seconds".format( e, timeout ) )
-        time.sleep( timeout )
-    raise thrown
-
-  return wrapper
-
-
-@WithRetry
-def UrlOpen( *args, **kwargs ):
-  return request.urlopen( *args, **kwargs )
-
-
-def DownloadFileTo( url,
-                    destination,
-                    file_name = None,
-                    checksum = None,
-                    check_certificate = True ):
-  if not file_name:
-    file_name = url.split( '/' )[ -1 ]
-
-  file_path = os.path.abspath( os.path.join( destination, file_name ) )
-
-  if not os.path.isdir( destination ):
-    os.makedirs( destination )
-
-  if os.path.exists( file_path ):
-    if checksum:
-      if ValidateCheckSumSHA256( file_path, checksum ):
-        print( "Checksum matches for {}, using it".format( file_path ) )
-        return file_path
-      else:
-        print( "Checksum doesn't match for {}, removing it".format(
-          file_path ) )
-
-    print( "Removing existing {}".format( file_path ) )
-    os.remove( file_path )
-
-  r = request.Request( url, headers = { 'User-Agent': 'Vimspector' } )
-
-  print( "Downloading {} to {}/{}".format( url, destination, file_name ) )
-
-  if not check_certificate:
-    context = ssl.create_default_context()
-    context.check_hostname = False
-    context.verify_mode = ssl.CERT_NONE
-    kwargs = { "context":  context }
-  else:
-    kwargs = {}
-
-  with contextlib.closing( UrlOpen( r, **kwargs ) ) as u:
-    with open( file_path, 'wb' ) as f:
-      f.write( u.read() )
-
-  if checksum:
-    if not ValidateCheckSumSHA256( file_path, checksum ):
-      raise RuntimeError(
-        'Checksum for {} ({}) does not match expected {}'.format(
-          file_path,
-          GetChecksumSHA254( file_path ),
-          checksum ) )
-  else:
-    print( "Checksum for {}: {}".format( file_path,
-                                         GetChecksumSHA254( file_path ) ) )
-
-  return file_path
-
-
-def GetChecksumSHA254( file_path ):
-  with open( file_path, 'rb' ) as existing_file:
-    return hashlib.sha256( existing_file.read() ).hexdigest()
-
-
-def ValidateCheckSumSHA256( file_path, checksum ):
-  existing_sha256 = GetChecksumSHA254( file_path )
-  return existing_sha256 == checksum
-
-
-def RemoveIfExists( destination ):
-  if os.path.exists( destination ) or os.path.islink( destination ):
-    if os.path.islink( destination ):
-      print( "Removing file {}".format( destination ) )
-      os.remove( destination )
-    else:
-      print( "Removing dir {}".format( destination ) )
-      shutil.rmtree( destination )
-
-
-# Python's ZipFile module strips execute bits from files, for no good reason
-# other than crappy code. Let's do it's job for it.
-class ModePreservingZipFile( zipfile.ZipFile ):
-  def extract( self, member, path = None, pwd = None ):
-    if not isinstance( member, zipfile.ZipInfo ):
-      member = self.getinfo( member )
-
-    if path is None:
-      path = os.getcwd()
-
-    ret_val = self._extract_member( member, path, pwd )
-    attr = member.external_attr >> 16
-    os.chmod( ret_val, attr )
-    return ret_val
-
-
-def ExtractZipTo( file_path, destination, format ):
-  print( "Extracting {} to {}".format( file_path, destination ) )
-  RemoveIfExists( destination )
-
-  if format == 'zip':
-    with ModePreservingZipFile( file_path ) as f:
-      f.extractall( path = destination )
-  elif format == 'zip.gz':
-    with gzip.open( file_path, 'rb' ) as f:
-      file_contents = f.read()
-
-    with ModePreservingZipFile( io.BytesIO( file_contents ) ) as f:
-      f.extractall( path = destination )
-
-  elif format == 'tar':
-    try:
-      with tarfile.open( file_path ) as f:
-        f.extractall( path = destination )
-    except Exception:
-      # There seems to a bug in python's tarfile that means it can't read some
-      # windows-generated tar files
-      os.makedirs( destination )
-      with CurrentWorkingDir( destination ):
-        subprocess.check_call( [ 'tar', 'zxvf', file_path ] )
-
-
-def MakeExtensionSymlink( name, root ):
-  MakeSymlink( gadget_dir, name, os.path.join( root, 'extension' ) ),
-
-
-def MakeSymlink( in_folder, link, pointing_to ):
-  RemoveIfExists( os.path.join( in_folder, link ) )
-
-  in_folder = os.path.abspath( in_folder )
-  pointing_to = os.path.relpath( os.path.abspath( pointing_to ),
-                                 in_folder )
-  os.symlink( pointing_to, os.path.join( in_folder, link ) )
-
-
-def CloneRepoTo( url, ref, destination ):
-  RemoveIfExists( destination )
-  git_in_repo = [ 'git', '-C', destination ]
-  subprocess.check_call( [ 'git', 'clone', url, destination ] )
-  subprocess.check_call( git_in_repo + [ 'checkout', ref ] )
-  subprocess.check_call( git_in_repo + [ 'submodule', 'sync', '--recursive' ] )
-  subprocess.check_call( git_in_repo + [ 'submodule',
-                                         'update',
-                                         '--init',
-                                         '--recursive' ] )
 
 
 def InstallGagdet( name, gadget, failed, all_adapters ):
@@ -682,7 +495,7 @@ def InstallGagdet( name, gadget, failed, all_adapters ):
 
       url = string.Template( gadget[ 'download' ][ 'url' ] ).substitute( v )
 
-      file_path = DownloadFileTo(
+      file_path = installer.DownloadFileTo(
         url,
         destination,
         file_name = gadget[ 'download' ].get( 'target' ),
@@ -690,21 +503,22 @@ def InstallGagdet( name, gadget, failed, all_adapters ):
         check_certificate = not args.no_check_certificate )
 
       root = os.path.join( destination, 'root' )
-      ExtractZipTo( file_path,
+      installer.ExtractZipTo(
-                    root,
+        file_path,
-                    format = gadget[ 'download' ].get( 'format', 'zip' ) )
+        root,
+        format = gadget[ 'download' ].get( 'format', 'zip' ) )
     elif 'repo' in gadget:
       url = string.Template( gadget[ 'repo' ][ 'url' ] ).substitute( v )
       ref = string.Template( gadget[ 'repo' ][ 'ref' ] ).substitute( v )
 
       destination = os.path.join( gadget_dir, 'download', name )
-      CloneRepoTo( url, ref, destination )
+      installer.CloneRepoTo( url, ref, destination )
       root = destination
 
     if 'do' in gadget:
       gadget[ 'do' ]( name, root, v )
     else:
-      MakeExtensionSymlink( name, root )
+      installer.MakeExtensionSymlink( vimspector_base, name, root )
 
     all_adapters.update( gadget.get( 'adapters', {} ) )
 

python3/vimspector/installer.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+
+# vimspector - A multi-language debugging system for Vim
+# Copyright 2019 Ben Jackson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib import request
+import io
+import contextlib
+import zipfile
+import gzip
+import shutil
+import tarfile
+import hashlib
+import time
+import ssl
+import subprocess
+import functools
+import os
+
+from vimspector import install
+
+
+@contextlib.contextmanager
+def CurrentWorkingDir( d ):
+  cur_d = os.getcwd()
+  try:
+    os.chdir( d )
+    yield
+  finally:
+    os.chdir( cur_d )
+
+
+def MakeExecutable( file_path ):
+  # TODO: import stat and use them by _just_ adding the X bit.
+  print( 'Making executable: {}'.format( file_path ) )
+  os.chmod( file_path, 0o755 )
+
+
+
+def WithRetry( f ):
+  retries = 5
+  timeout = 1 # seconds
+
+  @functools.wraps( f )
+  def wrapper( *args, **kwargs ):
+    thrown = None
+    for _ in range( retries ):
+      try:
+        return f( *args, **kwargs )
+      except Exception as e:
+        thrown = e
+        print( "Failed - {}, will retry in {} seconds".format( e, timeout ) )
+        time.sleep( timeout )
+    raise thrown
+
+  return wrapper
+
+
+@WithRetry
+def UrlOpen( *args, **kwargs ):
+  return request.urlopen( *args, **kwargs )
+
+
+def DownloadFileTo( url,
+                    destination,
+                    file_name = None,
+                    checksum = None,
+                    check_certificate = True ):
+  if not file_name:
+    file_name = url.split( '/' )[ -1 ]
+
+  file_path = os.path.abspath( os.path.join( destination, file_name ) )
+
+  if not os.path.isdir( destination ):
+    os.makedirs( destination )
+
+  if os.path.exists( file_path ):
+    if checksum:
+      if ValidateCheckSumSHA256( file_path, checksum ):
+        print( "Checksum matches for {}, using it".format( file_path ) )
+        return file_path
+      else:
+        print( "Checksum doesn't match for {}, removing it".format(
+          file_path ) )
+
+    print( "Removing existing {}".format( file_path ) )
+    os.remove( file_path )
+
+  r = request.Request( url, headers = { 'User-Agent': 'Vimspector' } )
+
+  print( "Downloading {} to {}/{}".format( url, destination, file_name ) )
+
+  if not check_certificate:
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+    kwargs = { "context":  context }
+  else:
+    kwargs = {}
+
+  with contextlib.closing( UrlOpen( r, **kwargs ) ) as u:
+    with open( file_path, 'wb' ) as f:
+      f.write( u.read() )
+
+  if checksum:
+    if not ValidateCheckSumSHA256( file_path, checksum ):
+      raise RuntimeError(
+        'Checksum for {} ({}) does not match expected {}'.format(
+          file_path,
+          GetChecksumSHA254( file_path ),
+          checksum ) )
+  else:
+    print( "Checksum for {}: {}".format( file_path,
+                                         GetChecksumSHA254( file_path ) ) )
+
+  return file_path
+
+
+def GetChecksumSHA254( file_path ):
+  with open( file_path, 'rb' ) as existing_file:
+    return hashlib.sha256( existing_file.read() ).hexdigest()
+
+
+def ValidateCheckSumSHA256( file_path, checksum ):
+  existing_sha256 = GetChecksumSHA254( file_path )
+  return existing_sha256 == checksum
+
+
+def RemoveIfExists( destination ):
+  if os.path.exists( destination ) or os.path.islink( destination ):
+    if os.path.islink( destination ):
+      print( "Removing file {}".format( destination ) )
+      os.remove( destination )
+    else:
+      print( "Removing dir {}".format( destination ) )
+      shutil.rmtree( destination )
+
+
+# Python's ZipFile module strips execute bits from files, for no good reason
+# other than crappy code. Let's do it's job for it.
+class ModePreservingZipFile( zipfile.ZipFile ):
+  def extract( self, member, path = None, pwd = None ):
+    if not isinstance( member, zipfile.ZipInfo ):
+      member = self.getinfo( member )
+
+    if path is None:
+      path = os.getcwd()
+
+    ret_val = self._extract_member( member, path, pwd )
+    attr = member.external_attr >> 16
+    os.chmod( ret_val, attr )
+    return ret_val
+
+
+def ExtractZipTo( file_path, destination, format ):
+  print( "Extracting {} to {}".format( file_path, destination ) )
+  RemoveIfExists( destination )
+
+  if format == 'zip':
+    with ModePreservingZipFile( file_path ) as f:
+      f.extractall( path = destination )
+  elif format == 'zip.gz':
+    with gzip.open( file_path, 'rb' ) as f:
+      file_contents = f.read()
+
+    with ModePreservingZipFile( io.BytesIO( file_contents ) ) as f:
+      f.extractall( path = destination )
+
+  elif format == 'tar':
+    try:
+      with tarfile.open( file_path ) as f:
+        f.extractall( path = destination )
+    except Exception:
+      # There seems to a bug in python's tarfile that means it can't read some
+      # windows-generated tar files
+      os.makedirs( destination )
+      with CurrentWorkingDir( destination ):
+        subprocess.check_call( [ 'tar', 'zxvf', file_path ] )
+
+
+def MakeExtensionSymlink( vimspector_base, name, root ):
+  MakeSymlink( install.GetGadgetDir( vimspector_base,
+                                     install.GetOS() ),
+               name,
+               os.path.join( root, 'extension' ) ),
+
+
+def MakeSymlink( in_folder, link, pointing_to ):
+  RemoveIfExists( os.path.join( in_folder, link ) )
+
+  in_folder = os.path.abspath( in_folder )
+  pointing_to = os.path.relpath( os.path.abspath( pointing_to ),
+                                 in_folder )
+  os.symlink( pointing_to, os.path.join( in_folder, link ) )
+
+
+def CloneRepoTo( url, ref, destination ):
+  RemoveIfExists( destination )
+  git_in_repo = [ 'git', '-C', destination ]
+  subprocess.check_call( [ 'git', 'clone', url, destination ] )
+  subprocess.check_call( git_in_repo + [ 'checkout', ref ] )
+  subprocess.check_call( git_in_repo + [ 'submodule', 'sync', '--recursive' ] )
+  subprocess.check_call( git_in_repo + [ 'submodule',
+                                         'update',
+                                         '--init',
+                                         '--recursive' ] )