From e37ef18c2812cbe2341a9c32dce613d485e0858e Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sun, 9 Feb 2020 16:08:55 +0000 Subject: [PATCH] Move installation routines into proper library --- install_gadget.py | 234 ++++---------------------------- python3/vimspector/installer.py | 218 +++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 210 deletions(-) create mode 100644 python3/vimspector/installer.py diff --git a/install_gadget.py b/install_gadget.py index 08738c1..561af21 100755 --- a/install_gadget.py +++ b/install_gadget.py @@ -21,23 +21,13 @@ if sys.version_info.major < 3: sys.exit( "You need to run this with python 3. Your version is " + '.'.join( map( str, sys.version_info[ :3 ] ) ) ) -from urllib import request import argparse -import contextlib import os import string -import zipfile -import gzip -import shutil import subprocess import traceback -import tarfile -import hashlib import json import functools -import time -import ssl -import io import operator import glob @@ -45,7 +35,7 @@ import glob sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), 'python3' ) ) -from vimspector import install +from vimspector import install, installer GADGETS = { 'vscode-cpptools': { @@ -210,7 +200,7 @@ GADGETS = { 'file_name': 'netcoredbg-linux-master.tar.gz', 'checksum': '', }, - 'do': lambda name, root, gadget: MakeSymlink( + 'do': lambda name, root, gadget: installer.MakeSymlink( gadget_dir, name, os.path.join( root, 'netcoredbg' ) ), @@ -393,29 +383,15 @@ GADGETS = { } -@contextlib.contextmanager -def CurrentWorkingDir( d ): - cur_d = os.getcwd() - try: - os.chdir( d ) - yield - finally: - os.chdir( cur_d ) - - -def MakeExecutable( file_path ): - # TODO: import stat and use them by _just_ adding the X bit. - print( 'Making executable: {}'.format( file_path ) ) - os.chmod( file_path, 0o755 ) - - def InstallCppTools( name, root, gadget ): extension = os.path.join( root, 'extension' ) # It's hilarious, but the execute bits aren't set in the vsix. So they # actually have javascript code which does this. It's just a horrible horrible # hack that really is not funny. - MakeExecutable( os.path.join( extension, 'debugAdapters', 'OpenDebugAD7' ) ) + installer.MakeExecutable( os.path.join( extension, + 'debugAdapters', + 'OpenDebugAD7' ) ) with open( os.path.join( extension, 'package.json' ) ) as f: package = json.load( f ) runtime_dependencies = package[ 'runtimeDependencies' ] @@ -423,14 +399,17 @@ def InstallCppTools( name, root, gadget ): for binary in dependency.get( 'binaries' ): file_path = os.path.abspath( os.path.join( extension, binary ) ) if os.path.exists( file_path ): - MakeExecutable( os.path.join( extension, binary ) ) + installer.MakeExecutable( os.path.join( extension, binary ) ) - MakeExtensionSymlink( name, root ) + installer.MakeExtensionSymlink( vimspector_base, name, root ) def InstallBashDebug( name, root, gadget ): - MakeExecutable( os.path.join( root, 'extension', 'bashdb_dir', 'bashdb' ) ) - MakeExtensionSymlink( name, root ) + installer.MakeExecutable( os.path.join( root, + 'extension', + 'bashdb_dir', + 'bashdb' ) ) + installer.MakeExtensionSymlink( vimspector_base, name, root ) def InstallDebugpy( name, root, gadget ): @@ -442,7 +421,7 @@ def InstallDebugpy( name, root, gadget ): finally: os.chdir( wd ) - MakeSymlink( gadget_dir, name, root ) + installer.MakeSymlink( gadget_dir, name, root ) def InstallTclProDebug( name, root, gadget ): @@ -473,11 +452,11 @@ def InstallTclProDebug( name, root, gadget ): break - with CurrentWorkingDir( os.path.join( root, 'lib', 'tclparser' ) ): + with installer.CurrentWorkingDir( os.path.join( root, 'lib', 'tclparser' ) ): subprocess.check_call( configure ) subprocess.check_call( [ 'make' ] ) - MakeSymlink( gadget_dir, name, root ) + installer.MakeSymlink( gadget_dir, name, root ) def InstallNodeDebug( name, root, gadget ): @@ -495,176 +474,10 @@ def InstallNodeDebug( name, root, gadget ): print( " $ ./install_gadget.py --enable-node ..." ) raise RuntimeError( 'Invalid node environent for node debugger' ) - with CurrentWorkingDir( root ): + with installer.CurrentWorkingDir( root ): subprocess.check_call( [ 'npm', 'install' ] ) subprocess.check_call( [ 'npm', 'run', 'build' ] ) - MakeSymlink( gadget_dir, name, root ) - - -def WithRetry( f ): - retries = 5 - timeout = 1 # seconds - - @functools.wraps( f ) - def wrapper( *args, **kwargs ): - thrown = None - for _ in range( retries ): - try: - return f( *args, **kwargs ) - except Exception as e: - thrown = e - print( "Failed - {}, will retry in {} seconds".format( e, timeout ) ) - time.sleep( timeout ) - raise thrown - - return wrapper - - -@WithRetry -def UrlOpen( *args, **kwargs ): - return request.urlopen( *args, **kwargs ) - - -def DownloadFileTo( url, - destination, - file_name = None, - checksum = None, - check_certificate = True ): - if not file_name: - file_name = url.split( '/' )[ -1 ] - - file_path = os.path.abspath( os.path.join( destination, file_name ) ) - - if not os.path.isdir( destination ): - os.makedirs( destination ) - - if os.path.exists( file_path ): - if checksum: - if ValidateCheckSumSHA256( file_path, checksum ): - print( "Checksum matches for {}, using it".format( file_path ) ) - return file_path - else: - print( "Checksum doesn't match for {}, removing it".format( - file_path ) ) - - print( "Removing existing {}".format( file_path ) ) - os.remove( file_path ) - - r = request.Request( url, headers = { 'User-Agent': 'Vimspector' } ) - - print( "Downloading {} to {}/{}".format( url, destination, file_name ) ) - - if not check_certificate: - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - kwargs = { "context": context } - else: - kwargs = {} - - with contextlib.closing( UrlOpen( r, **kwargs ) ) as u: - with open( file_path, 'wb' ) as f: - f.write( u.read() ) - - if checksum: - if not ValidateCheckSumSHA256( file_path, checksum ): - raise RuntimeError( - 'Checksum for {} ({}) does not match expected {}'.format( - file_path, - GetChecksumSHA254( file_path ), - checksum ) ) - else: - print( "Checksum for {}: {}".format( file_path, - GetChecksumSHA254( file_path ) ) ) - - return file_path - - -def GetChecksumSHA254( file_path ): - with open( file_path, 'rb' ) as existing_file: - return hashlib.sha256( existing_file.read() ).hexdigest() - - -def ValidateCheckSumSHA256( file_path, checksum ): - existing_sha256 = GetChecksumSHA254( file_path ) - return existing_sha256 == checksum - - -def RemoveIfExists( destination ): - if os.path.exists( destination ) or os.path.islink( destination ): - if os.path.islink( destination ): - print( "Removing file {}".format( destination ) ) - os.remove( destination ) - else: - print( "Removing dir {}".format( destination ) ) - shutil.rmtree( destination ) - - -# Python's ZipFile module strips execute bits from files, for no good reason -# other than crappy code. Let's do it's job for it. -class ModePreservingZipFile( zipfile.ZipFile ): - def extract( self, member, path = None, pwd = None ): - if not isinstance( member, zipfile.ZipInfo ): - member = self.getinfo( member ) - - if path is None: - path = os.getcwd() - - ret_val = self._extract_member( member, path, pwd ) - attr = member.external_attr >> 16 - os.chmod( ret_val, attr ) - return ret_val - - -def ExtractZipTo( file_path, destination, format ): - print( "Extracting {} to {}".format( file_path, destination ) ) - RemoveIfExists( destination ) - - if format == 'zip': - with ModePreservingZipFile( file_path ) as f: - f.extractall( path = destination ) - elif format == 'zip.gz': - with gzip.open( file_path, 'rb' ) as f: - file_contents = f.read() - - with ModePreservingZipFile( io.BytesIO( file_contents ) ) as f: - f.extractall( path = destination ) - - elif format == 'tar': - try: - with tarfile.open( file_path ) as f: - f.extractall( path = destination ) - except Exception: - # There seems to a bug in python's tarfile that means it can't read some - # windows-generated tar files - os.makedirs( destination ) - with CurrentWorkingDir( destination ): - subprocess.check_call( [ 'tar', 'zxvf', file_path ] ) - - -def MakeExtensionSymlink( name, root ): - MakeSymlink( gadget_dir, name, os.path.join( root, 'extension' ) ), - - -def MakeSymlink( in_folder, link, pointing_to ): - RemoveIfExists( os.path.join( in_folder, link ) ) - - in_folder = os.path.abspath( in_folder ) - pointing_to = os.path.relpath( os.path.abspath( pointing_to ), - in_folder ) - os.symlink( pointing_to, os.path.join( in_folder, link ) ) - - -def CloneRepoTo( url, ref, destination ): - RemoveIfExists( destination ) - git_in_repo = [ 'git', '-C', destination ] - subprocess.check_call( [ 'git', 'clone', url, destination ] ) - subprocess.check_call( git_in_repo + [ 'checkout', ref ] ) - subprocess.check_call( git_in_repo + [ 'submodule', 'sync', '--recursive' ] ) - subprocess.check_call( git_in_repo + [ 'submodule', - 'update', - '--init', - '--recursive' ] ) + installer.MakeSymlink( gadget_dir, name, root ) def InstallGagdet( name, gadget, failed, all_adapters ): @@ -682,7 +495,7 @@ def InstallGagdet( name, gadget, failed, all_adapters ): url = string.Template( gadget[ 'download' ][ 'url' ] ).substitute( v ) - file_path = DownloadFileTo( + file_path = installer.DownloadFileTo( url, destination, file_name = gadget[ 'download' ].get( 'target' ), @@ -690,21 +503,22 @@ def InstallGagdet( name, gadget, failed, all_adapters ): check_certificate = not args.no_check_certificate ) root = os.path.join( destination, 'root' ) - ExtractZipTo( file_path, - root, - format = gadget[ 'download' ].get( 'format', 'zip' ) ) + installer.ExtractZipTo( + file_path, + root, + format = gadget[ 'download' ].get( 'format', 'zip' ) ) elif 'repo' in gadget: url = string.Template( gadget[ 'repo' ][ 'url' ] ).substitute( v ) ref = string.Template( gadget[ 'repo' ][ 'ref' ] ).substitute( v ) destination = os.path.join( gadget_dir, 'download', name ) - CloneRepoTo( url, ref, destination ) + installer.CloneRepoTo( url, ref, destination ) root = destination if 'do' in gadget: gadget[ 'do' ]( name, root, v ) else: - MakeExtensionSymlink( name, root ) + installer.MakeExtensionSymlink( vimspector_base, name, root ) all_adapters.update( gadget.get( 'adapters', {} ) ) diff --git a/python3/vimspector/installer.py b/python3/vimspector/installer.py new file mode 100644 index 0000000..53a0150 --- /dev/null +++ b/python3/vimspector/installer.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 + +# vimspector - A multi-language debugging system for Vim +# Copyright 2019 Ben Jackson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from urllib import request +import io +import contextlib +import zipfile +import gzip +import shutil +import tarfile +import hashlib +import time +import ssl +import subprocess +import functools +import os + +from vimspector import install + + +@contextlib.contextmanager +def CurrentWorkingDir( d ): + cur_d = os.getcwd() + try: + os.chdir( d ) + yield + finally: + os.chdir( cur_d ) + + +def MakeExecutable( file_path ): + # TODO: import stat and use them by _just_ adding the X bit. + print( 'Making executable: {}'.format( file_path ) ) + os.chmod( file_path, 0o755 ) + + + +def WithRetry( f ): + retries = 5 + timeout = 1 # seconds + + @functools.wraps( f ) + def wrapper( *args, **kwargs ): + thrown = None + for _ in range( retries ): + try: + return f( *args, **kwargs ) + except Exception as e: + thrown = e + print( "Failed - {}, will retry in {} seconds".format( e, timeout ) ) + time.sleep( timeout ) + raise thrown + + return wrapper + + +@WithRetry +def UrlOpen( *args, **kwargs ): + return request.urlopen( *args, **kwargs ) + + +def DownloadFileTo( url, + destination, + file_name = None, + checksum = None, + check_certificate = True ): + if not file_name: + file_name = url.split( '/' )[ -1 ] + + file_path = os.path.abspath( os.path.join( destination, file_name ) ) + + if not os.path.isdir( destination ): + os.makedirs( destination ) + + if os.path.exists( file_path ): + if checksum: + if ValidateCheckSumSHA256( file_path, checksum ): + print( "Checksum matches for {}, using it".format( file_path ) ) + return file_path + else: + print( "Checksum doesn't match for {}, removing it".format( + file_path ) ) + + print( "Removing existing {}".format( file_path ) ) + os.remove( file_path ) + + r = request.Request( url, headers = { 'User-Agent': 'Vimspector' } ) + + print( "Downloading {} to {}/{}".format( url, destination, file_name ) ) + + if not check_certificate: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + kwargs = { "context": context } + else: + kwargs = {} + + with contextlib.closing( UrlOpen( r, **kwargs ) ) as u: + with open( file_path, 'wb' ) as f: + f.write( u.read() ) + + if checksum: + if not ValidateCheckSumSHA256( file_path, checksum ): + raise RuntimeError( + 'Checksum for {} ({}) does not match expected {}'.format( + file_path, + GetChecksumSHA254( file_path ), + checksum ) ) + else: + print( "Checksum for {}: {}".format( file_path, + GetChecksumSHA254( file_path ) ) ) + + return file_path + + +def GetChecksumSHA254( file_path ): + with open( file_path, 'rb' ) as existing_file: + return hashlib.sha256( existing_file.read() ).hexdigest() + + +def ValidateCheckSumSHA256( file_path, checksum ): + existing_sha256 = GetChecksumSHA254( file_path ) + return existing_sha256 == checksum + + +def RemoveIfExists( destination ): + if os.path.exists( destination ) or os.path.islink( destination ): + if os.path.islink( destination ): + print( "Removing file {}".format( destination ) ) + os.remove( destination ) + else: + print( "Removing dir {}".format( destination ) ) + shutil.rmtree( destination ) + + +# Python's ZipFile module strips execute bits from files, for no good reason +# other than crappy code. Let's do it's job for it. +class ModePreservingZipFile( zipfile.ZipFile ): + def extract( self, member, path = None, pwd = None ): + if not isinstance( member, zipfile.ZipInfo ): + member = self.getinfo( member ) + + if path is None: + path = os.getcwd() + + ret_val = self._extract_member( member, path, pwd ) + attr = member.external_attr >> 16 + os.chmod( ret_val, attr ) + return ret_val + + +def ExtractZipTo( file_path, destination, format ): + print( "Extracting {} to {}".format( file_path, destination ) ) + RemoveIfExists( destination ) + + if format == 'zip': + with ModePreservingZipFile( file_path ) as f: + f.extractall( path = destination ) + elif format == 'zip.gz': + with gzip.open( file_path, 'rb' ) as f: + file_contents = f.read() + + with ModePreservingZipFile( io.BytesIO( file_contents ) ) as f: + f.extractall( path = destination ) + + elif format == 'tar': + try: + with tarfile.open( file_path ) as f: + f.extractall( path = destination ) + except Exception: + # There seems to a bug in python's tarfile that means it can't read some + # windows-generated tar files + os.makedirs( destination ) + with CurrentWorkingDir( destination ): + subprocess.check_call( [ 'tar', 'zxvf', file_path ] ) + + +def MakeExtensionSymlink( vimspector_base, name, root ): + MakeSymlink( install.GetGadgetDir( vimspector_base, + install.GetOS() ), + name, + os.path.join( root, 'extension' ) ), + + +def MakeSymlink( in_folder, link, pointing_to ): + RemoveIfExists( os.path.join( in_folder, link ) ) + + in_folder = os.path.abspath( in_folder ) + pointing_to = os.path.relpath( os.path.abspath( pointing_to ), + in_folder ) + os.symlink( pointing_to, os.path.join( in_folder, link ) ) + + +def CloneRepoTo( url, ref, destination ): + RemoveIfExists( destination ) + git_in_repo = [ 'git', '-C', destination ] + subprocess.check_call( [ 'git', 'clone', url, destination ] ) + subprocess.check_call( git_in_repo + [ 'checkout', ref ] ) + subprocess.check_call( git_in_repo + [ 'submodule', 'sync', '--recursive' ] ) + subprocess.check_call( git_in_repo + [ 'submodule', + 'update', + '--init', + '--recursive' ] )