From d29b0ebcfd2f86afc027759159c688599ce946b6 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Tue, 22 Oct 2019 23:33:38 +0100 Subject: [PATCH 1/2] Fix installation issues --- azure-pipelines.yml | 14 +++++++++----- install_gadget.py | 35 ++++++++++++++++++++++++----------- tests/ci/image/Dockerfile | 3 ++- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d7b088c..6976b6c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,8 +42,9 @@ stages: go get -u github.com/go-delve/delve/cmd/dlv displayName: 'Install Delve for Go' - - bash: python3 install_gadget.py --all - displayName: 'Install gadgets' + # TODO: Use cache to store the download directory + - bash: python3 install_gadget.py --all --force-all + displayName: 'Install gadgets - python3' - bash: vim --version displayName: 'Print vim version information' @@ -68,14 +69,17 @@ stages: pool: vmImage: 'macOS-10.13' steps: - - bash: brew unlink node@6 && brew install macvim node + - bash: | + brew unlink node@6 + brew install macvim node@10 + brew link --force --overwrite node@10 displayName: 'Install vim and node' - bash: go get -u github.com/go-delve/delve/cmd/dlv displayName: 'Install Delve for Go' - - bash: python3 install_gadget.py --all - displayName: 'Install gadgets' + - bash: python3 install_gadget.py --all --force-all + displayName: 'Install gadgets - python3' - bash: vim --version displayName: 'Print vim version information' diff --git a/install_gadget.py b/install_gadget.py index 88606ac..5037018 100755 --- a/install_gadget.py +++ b/install_gadget.py @@ -25,6 +25,7 @@ import contextlib import os import string import zipfile +import gzip import shutil import subprocess import traceback @@ -33,6 +34,11 @@ import hashlib import sys import json +try: + from io import BytesIO ## for Python 3 +except ImportError: + from BytesIO import BytesIO + # Include vimspector source, for utils sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), 'python3' ) ) @@ -150,8 +156,8 @@ GADGETS = { 'url': 'https://marketplace.visualstudio.com/_apis/public/gallery/' 'publishers/ms-vscode/vsextensions/mono-debug/${version}/' 'vspackage', - 'target': 'vscode-mono-debug.tar.gz', - 'format': 'tar', + 'target': 'vscode-mono-debug.vsix.gz', + 'format': 'zip.gz', }, 'all': { 'file_name': 'vscode-mono-debug.vsix', @@ -226,19 +232,20 @@ GADGETS = { }, }, 'debugger-for-chrome': { - 'language': 'typescript', + 'language': 'chrome', 'enabled': False, 'download': { 'url': 'https://marketplace.visualstudio.com/_apis/public/gallery/' 'publishers/msjsdiag/vsextensions/' 'debugger-for-chrome/${version}/vspackage', - 'target': 'msjsdiag.debugger-for-chrome-4.12.0.tar.gz', - 'format': 'tar', + 'target': 'msjsdiag.debugger-for-chrome-4.12.0.vsix.gz', + 'format': 'zip.gz', }, 'all': { 'version': '4.12.0', 'file_name': 'msjsdiag.debugger-for-chrome-4.12.0.vsix', - 'checksum': '' + 'checksum': + '0df2fe96d059a002ebb0936b0003e6569e5a5c35260dc3791e1657d27d82ccf5' }, 'adapters': { 'chrome': { @@ -275,7 +282,7 @@ def InstallCppTools( name, root ): # It's hilarious, but the execute bits aren't set in the vsix. So they # actually have javascript code which does this. It's just a horrible horrible - # hoke that really is not funny. + # hack that really is not funny. MakeExecutable( os.path.join( extension, 'debugAdapters', 'OpenDebugAD7' ) ) with open( os.path.join( extension, 'package.json' ) ) as f: package = json.load( f ) @@ -325,9 +332,10 @@ def InstallTclProDebug( name, root ): def InstallNodeDebug( name, root ): - node_version = subprocess.check_output( [ 'node', '--version' ] ).strip() + node_version = subprocess.check_output( [ 'node', '--version' ], + universal_newlines=True ).strip() print( "Node.js version: {}".format( node_version ) ) - if map( int, node_version[ 1: ].split( '.' ) ) >= [ 12, 0, 0 ]: + if list( map( int, node_version[ 1: ].split( '.' ) ) ) >= [ 12, 0, 0 ]: print( "Can't install vscode-debug-node2:" ) print( "Sorry, you appear to be running node 12 or later. That's not " "compatible with the build system for this extension, and as far as " @@ -365,7 +373,6 @@ def DownloadFileTo( url, destination, file_name = None, checksum = None ): print( "Removing existing {}".format( file_path ) ) os.remove( file_path ) - r = urllib2.Request( url, headers = { 'User-Agent': 'Vimspector' } ) print( "Downloading {} to {}/{}".format( url, destination, file_name ) ) @@ -431,7 +438,13 @@ def ExtractZipTo( file_path, destination, format ): if format == 'zip': with ModePreservingZipFile( file_path ) as f: f.extractall( path = destination ) - return + elif format == 'zip.gz': + with gzip.open( file_path, 'rb' ) as f: + file_contents = f.read() + + with ModePreservingZipFile( BytesIO( file_contents ) ) as f: + f.extractall( path = destination ) + elif format == 'tar': try: with tarfile.open( file_path ) as f: diff --git a/tests/ci/image/Dockerfile b/tests/ci/image/Dockerfile index 280d179..71329b6 100644 --- a/tests/ci/image/Dockerfile +++ b/tests/ci/image/Dockerfile @@ -17,7 +17,8 @@ RUN apt-get update && \ gdb \ lldb \ curl \ - nodejs && \ + nodejs \ + npm && \ apt-get -y autoremove RUN ln -fs /usr/share/zoneinfo/Europe/London /etc/localtime && \ From 1a77b32e011a32cc0c7ba89d57864b83834fbdce Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Wed, 23 Oct 2019 00:05:21 +0100 Subject: [PATCH 2/2] Flaky CI seems to fail to download sometimes --- install_gadget.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/install_gadget.py b/install_gadget.py index 5037018..eaaadbe 100755 --- a/install_gadget.py +++ b/install_gadget.py @@ -33,6 +33,8 @@ import tarfile import hashlib import sys import json +import functools +import time try: from io import BytesIO ## for Python 3 @@ -352,6 +354,30 @@ def InstallNodeDebug( name, root ): MakeSymlink( gadget_dir, name, root ) +def WithRetry( f ): + retries = 5 + timeout = 1 # seconds + + @functools.wraps( f ) + def wrapper( *args, **kwargs ): + thrown = None + for _ in range( retries ): + try: + return f( *args, **kwargs ) + except Exception as e: + thrown = e + print( "Failed - {}, will retry in {} seconds".format( e, timeout ) ) + time.sleep( timeout ) + raise thrown + + return wrapper + + +@WithRetry +def UrlOpen( *args, **kwargs ): + return urllib2.urlopen( *args, **kwargs ) + + def DownloadFileTo( url, destination, file_name = None, checksum = None ): if not file_name: file_name = url.split( '/' )[ -1 ] @@ -377,7 +403,7 @@ def DownloadFileTo( url, destination, file_name = None, checksum = None ): print( "Downloading {} to {}/{}".format( url, destination, file_name ) ) - with contextlib.closing( urllib2.urlopen( r ) ) as u: + with contextlib.closing( UrlOpen( r ) ) as u: with open( file_path, 'wb' ) as f: f.write( u.read() )