From 3bfbab1dae8937514867abe35b20ef9365328eda Mon Sep 17 00:00:00 2001 From: d3v53c Date: Mon, 14 Sep 2020 04:28:19 -0700 Subject: [PATCH 1/7] cmd exec using subprocess rather than system calls --- Tools/mkdist.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Tools/mkdist.py b/Tools/mkdist.py index 7116144a0..92c12d88b 100755 --- a/Tools/mkdist.py +++ b/Tools/mkdist.py @@ -8,6 +8,12 @@ def failed(): print("mkdist.py failed to complete") sys.exit(2) +def check_file_exists(path): + return os.path.isfile(path) + +def check_dir_exists(path): + return os.path.isdir(path) + import argparse parser = argparse.ArgumentParser(description="Build a SWIG distribution tarball swig-x.y.z.tar.gz") parser.add_argument("version", help="version string in format x.y.z") @@ -33,13 +39,18 @@ if dirname.lower() != dirname: # If directory and tarball exist, remove it print("Removing " + dirname) -os.system("rm -rf " + dirname) +if check_dir_exists(dirname): + subprocess.call(["rm", "-rf", dirname]) print("Removing " + dirname + ".tar if exists") -os.system("rm -f " + dirname + ".tar.gz") +filename = dirname + ".tar" +if check_file_exists(filename): + subprocess.call(["rm", "-rf", filename]) print("Removing " + dirname + ".tar.gz if exists") -os.system("rm -f " + dirname + ".tar") +filename += ".gz" +if check_file_exists(filename): + subprocess.call(["rm", "-rf", filename]) # Grab the code from git From 2a4b4ea6e691510247506acecaa2f575cb8ea278 Mon Sep 17 00:00:00 2001 From: "Mr. Krabbs" Date: Mon, 14 Sep 2020 14:44:48 -0700 Subject: [PATCH 2/7] changed os system calls to subprocess calls --- Tools/mkdist.py | 77 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/Tools/mkdist.py b/Tools/mkdist.py index 92c12d88b..187a55d6e 100755 --- a/Tools/mkdist.py +++ b/Tools/mkdist.py @@ -9,11 +9,23 @@ def failed(): sys.exit(2) def check_file_exists(path): + """ + Checks if a file exists or not. + """ return os.path.isfile(path) def check_dir_exists(path): + """ + Checks if a folder exists or not. + """ return os.path.isdir(path) +def run_command(*args, **kwargs): + """ + Runs an os command using subprocess module. + """ + return subprocess.call([*args], **kwargs) + import argparse parser = argparse.ArgumentParser(description="Build a SWIG distribution tarball swig-x.y.z.tar.gz") parser.add_argument("version", help="version string in format x.y.z") @@ -28,6 +40,13 @@ dirname = "swig-" + version force_tag = args.force_tag skip_checks = args.skip_checks +# Tools directory path $ENV/swig/Tools +toolsdir = os.path.dirname(os.path.abspath(__file__)) +# Root directory path (swig) $ENV/swig +rootdir = os.path.abspath(os.path.join(toolsdir, os.pardir)) +# version directory path $ENV/swig/ +dirpath = os.path.join(rootdir, dirname) + if sys.version_info[0:2] < (2, 7): print("Error: Python 2.7 or higher is required") sys.exit(3) @@ -39,26 +58,26 @@ if dirname.lower() != dirname: # If directory and tarball exist, remove it print("Removing " + dirname) -if check_dir_exists(dirname): - subprocess.call(["rm", "-rf", dirname]) +if check_dir_exists(dirpath): + run_command("rm", "-rf", dirpath) print("Removing " + dirname + ".tar if exists") -filename = dirname + ".tar" +filename = dirpath + ".tar" if check_file_exists(filename): - subprocess.call(["rm", "-rf", filename]) + run_command("rm", "-rf", filename) print("Removing " + dirname + ".tar.gz if exists") filename += ".gz" if check_file_exists(filename): - subprocess.call(["rm", "-rf", filename]) + run_command("rm", "-rf", filename) # Grab the code from git print("Checking there are no local changes in git repo") -os.system("git remote update origin") == 0 or failed() +run_command("git", "remote", "update", "origin") == 0 or failed() command = ["git", "status", "--porcelain", "-uno"] out = subprocess.check_output(command) -if out.strip() != "": +if out.strip(): print("Local git repository has modifications") print(" ".join(command)) print(out) @@ -68,7 +87,7 @@ if not skip_checks: print("Checking git repository is in sync with remote repository") command = ["git", "log", "--oneline", branch + "..origin/" + branch] out = subprocess.check_output(command) - if out.strip() != "": + if out.strip(): print("Remote repository has additional modifications to local repository") print(" ".join(command)) print(out) @@ -76,7 +95,7 @@ if not skip_checks: command = ["git", "log", "--oneline", "origin/" + branch + ".." + branch] out = subprocess.check_output(command) - if out.strip() != "": + if out.strip(): print("Local repository has modifications not pushed to the remote repository") print("These should be pushed and checked that they pass Continuous Integration testing before continuing") print(" ".join(command)) @@ -84,31 +103,49 @@ if not skip_checks: sys.exit(3) print("Tagging release") -tag = "'v" + version + "'" +tag = "v" + version force = "-f " if force_tag else "" -os.system("git tag -a -m 'Release version " + version + "' " + force + tag) == 0 or failed() +command = ["git", "tag", "-a", "-m", "'Release version " + version + "'"] +force and command.extend(force, tag) +not force and command.append(tag) +run_command(*command) == 0 or failed() -outdir = os.path.basename(os.getcwd()) + "/" + dirname + "/" +outdir = dirname + "/" print("Grabbing tagged release git repository using 'git archive' into " + outdir) -os.system("(cd .. && git archive --prefix=" + outdir + " " + tag + " . | tar -xf -)") == 0 or failed() + +# using pipe operator without shell=True; split commands into individual ones. +# git archive command +command = ["git", "archive", "--prefix=" + outdir, tag, "."] +archive_ps = subprocess.Popen((*command, ), cwd=rootdir, stdout=subprocess.PIPE) +# tar -xf - +tar_ps = subprocess.Popen(("tar", "-xf", "-"), stdin=archive_ps.stdout, stdout=subprocess.PIPE) +archive_ps.stdout.close() # Allow archive_ps to receive a SIGPIPE if tar_ps exits. +output = tar_ps.communicate() # Go build the system print("Building system") -os.system("cd " + dirname + " && ./autogen.sh") == 0 or failed() -os.system("cd " + dirname + "/Source/CParse && bison -y -d parser.y && mv y.tab.c parser.c && mv y.tab.h parser.h") == 0 or failed() -os.system("cd " + dirname + " && make -f Makefile.in libfiles srcdir=./") == 0 or failed() +run_command("./autogen.sh", cwd=dirpath) == 0 or failed() + +cmdpath = os.path.join(dirpath, "Source", "CParse") +run_command("bison", "-y", "-d", "parser.y", cwd=cmdpath) == 0 or failed() +run_command("mv", "y.tab.c", "parser.c", cwd=cmdpath) == 0 or failed() +run_command("mv", "y.tab.h", "parser.h", cwd=cmdpath) == 0 or failed() + +run_command("make", "-f", "Makefile.in", "libfiles", "srcdir=./", cwd=dirpath) == 0 or failed() # Remove autoconf files -os.system("find " + dirname + " -name autom4te.cache -exec rm -rf {} \\;") +run_command("find", dirname, "-name", "autom4te.cache", "-exec", "rm", "-rf", "{}", ";", cwd=rootdir) # Build documentation print("Building html documentation") -os.system("cd " + dirname + "/Doc/Manual && make all clean-baks") == 0 or failed() +# os.system("cd " + dirname + "/Doc/Manual && make all clean-baks") == 0 or failed() +docpath = os.path.join(dirpath, "Doc", "Manual") +run_command("make", "all", "clean-baks", cwd=docpath) == 0 or failed() # Build the tar-ball -os.system("tar -cf " + dirname + ".tar " + dirname) == 0 or failed() -os.system("gzip " + dirname + ".tar") == 0 or failed() +run_command("tar", "-cf", dirname + ".tar", dirname, stdout=open(dirname + ".tar", "w")) == 0 or failed() +run_command("gzip", dirname + ".tar", stdout=open(dirname + ".tar.gz", "w")) == 0 or failed() print("Finished building " + dirname + ".tar.gz") From 08478798f177c2f4be5ad4d717ba1c02ad45c1b0 Mon Sep 17 00:00:00 2001 From: "Mr. Krabbs" Date: Mon, 14 Sep 2020 15:18:11 -0700 Subject: [PATCH 3/7] cleanup --- Tools/mkdist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/mkdist.py b/Tools/mkdist.py index 187a55d6e..2d9e029f1 100755 --- a/Tools/mkdist.py +++ b/Tools/mkdist.py @@ -139,7 +139,6 @@ run_command("find", dirname, "-name", "autom4te.cache", "-exec", "rm", "-rf", "{ # Build documentation print("Building html documentation") -# os.system("cd " + dirname + "/Doc/Manual && make all clean-baks") == 0 or failed() docpath = os.path.join(dirpath, "Doc", "Manual") run_command("make", "all", "clean-baks", cwd=docpath) == 0 or failed() From ca567a95576a9af601451eff76dbb1002805f63e Mon Sep 17 00:00:00 2001 From: "Mr. Krabbs" Date: Wed, 16 Sep 2020 07:29:37 -0700 Subject: [PATCH 4/7] removed destructuring operator for backward compatibililty --- Tools/mkdist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/mkdist.py b/Tools/mkdist.py index 2d9e029f1..1a9d00be2 100755 --- a/Tools/mkdist.py +++ b/Tools/mkdist.py @@ -24,7 +24,7 @@ def run_command(*args, **kwargs): """ Runs an os command using subprocess module. """ - return subprocess.call([*args], **kwargs) + return subprocess.call(args, **kwargs) import argparse parser = argparse.ArgumentParser(description="Build a SWIG distribution tarball swig-x.y.z.tar.gz") @@ -116,7 +116,7 @@ print("Grabbing tagged release git repository using 'git archive' into " + outdi # using pipe operator without shell=True; split commands into individual ones. # git archive command command = ["git", "archive", "--prefix=" + outdir, tag, "."] -archive_ps = subprocess.Popen((*command, ), cwd=rootdir, stdout=subprocess.PIPE) +archive_ps = subprocess.Popen(command, cwd=rootdir, stdout=subprocess.PIPE) # tar -xf - tar_ps = subprocess.Popen(("tar", "-xf", "-"), stdin=archive_ps.stdout, stdout=subprocess.PIPE) archive_ps.stdout.close() # Allow archive_ps to receive a SIGPIPE if tar_ps exits. From a05fc7931069e39b0194aaefcaa96014cb11f4d6 Mon Sep 17 00:00:00 2001 From: Jamie Slome Date: Thu, 25 Mar 2021 06:57:33 +0000 Subject: [PATCH 5/7] Update mkdist.py --- Tools/mkdist.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/Tools/mkdist.py b/Tools/mkdist.py index 1a9d00be2..4fba4701d 100755 --- a/Tools/mkdist.py +++ b/Tools/mkdist.py @@ -3,29 +3,12 @@ import sys import os import subprocess +from utils import * def failed(): print("mkdist.py failed to complete") sys.exit(2) -def check_file_exists(path): - """ - Checks if a file exists or not. - """ - return os.path.isfile(path) - -def check_dir_exists(path): - """ - Checks if a folder exists or not. - """ - return os.path.isdir(path) - -def run_command(*args, **kwargs): - """ - Runs an os command using subprocess module. - """ - return subprocess.call(args, **kwargs) - import argparse parser = argparse.ArgumentParser(description="Build a SWIG distribution tarball swig-x.y.z.tar.gz") parser.add_argument("version", help="version string in format x.y.z") @@ -44,8 +27,10 @@ skip_checks = args.skip_checks toolsdir = os.path.dirname(os.path.abspath(__file__)) # Root directory path (swig) $ENV/swig rootdir = os.path.abspath(os.path.join(toolsdir, os.pardir)) +# current directory +current_dir = os.getcwd() # version directory path $ENV/swig/ -dirpath = os.path.join(rootdir, dirname) +dirpath = os.path.join(current_dir, dirname) if sys.version_info[0:2] < (2, 7): print("Error: Python 2.7 or higher is required") @@ -125,6 +110,7 @@ output = tar_ps.communicate() # Go build the system print("Building system") +run_command("mkdir", "-p", dirpath) run_command("./autogen.sh", cwd=dirpath) == 0 or failed() cmdpath = os.path.join(dirpath, "Source", "CParse") From 8852a1b6d14ba9d6ab23f7cf8be32830040029a1 Mon Sep 17 00:00:00 2001 From: Jamie Slome Date: Thu, 25 Mar 2021 06:58:11 +0000 Subject: [PATCH 6/7] Update mkrelease.py --- Tools/mkrelease.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Tools/mkrelease.py b/Tools/mkrelease.py index 014cef227..28dca265b 100755 --- a/Tools/mkrelease.py +++ b/Tools/mkrelease.py @@ -2,6 +2,7 @@ import sys import os +from utils import * def failed(message): if message == "": @@ -27,15 +28,17 @@ skip_checks = args.skip_checks username = args.username print("Looking for rsync") -os.system("which rsync") and failed("rsync not installed/found. Please install.") +run_command("which", "rsync") and failed("rsync not installed/found. Please install.") print("Making source tarball") force = "--force-tag" if force_tag else "" skip = "--skip-checks" if skip_checks else "" -os.system("python ./mkdist.py {} {} --branch {} {}".format(force, skip, branch, version)) and failed("") +toolsdir = os.path.dirname(os.path.abspath(__file__)) +cmd = "python {}/mkdist.py {} {} --branch {} {}".format(toolsdir, force, skip, branch, version).split() +run_command(*cmd) and failed("") print("Build Windows package") -os.system("./mkwindows.sh " + version) and failed("") +run_command("{}/mkwindows.sh".format(toolsdir), version) and failed("") if username: print("Uploading to SourceForge") @@ -45,11 +48,11 @@ if username: # If a file with 'readme' in the name exists in the same folder as the zip/tarball, it gets automatically displayed as the release notes by SF full_readme_file = "readme-" + version + ".txt" - os.system("rm -f " + full_readme_file) - os.system("cat swig-" + version + "/README " + "swig-" + version + "/CHANGES.current " + "swig-" + version + "/RELEASENOTES " + "> " + full_readme_file) + run_command("rm", "-f", full_readme_file) + run_command("cat", "swig-" + version + "/README", "swig-" + version + "/CHANGES.current", "swig-" + version + "/RELEASENOTES", ">", full_readme_file) - os.system("rsync --archive --verbose -P --times -e ssh " + "swig-" + version + ".tar.gz " + full_readme_file + " " + swig_dir_sf) and failed("") - os.system("rsync --archive --verbose -P --times -e ssh " + "swigwin-" + version + ".zip " + full_readme_file + " " + swigwin_dir_sf) and failed("") + run_command("rsync", "--archive", "--verbose", "-P", "--times", "-e", "ssh", "swig-" + version + ".tar.gz", full_readme_file, swig_dir_sf) and failed("") + run_command("rsync", "--archive", "--verbose", "-P", "--times", "-e", "ssh", "swigwin-" + version + ".zip", full_readme_file, swigwin_dir_sf) and failed("") print("Finished") From 0fc44beac5c0a940d1ea4cd091eeaeaa6fd21fe2 Mon Sep 17 00:00:00 2001 From: Jamie Slome Date: Thu, 25 Mar 2021 06:59:01 +0000 Subject: [PATCH 7/7] Create utils.py --- Tools/utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Tools/utils.py diff --git a/Tools/utils.py b/Tools/utils.py new file mode 100644 index 000000000..496882f19 --- /dev/null +++ b/Tools/utils.py @@ -0,0 +1,26 @@ +import os, subprocess + + +def check_file_exists(path): + """ + Checks if a file exists or not. + """ + return os.path.isfile(path) + + +def check_dir_exists(path): + """ + Checks if a folder exists or not. + """ + return os.path.isdir(path) + + +def run_command(*args, **kwargs): + """ + Runs an os command using subprocess module. + """ + redirect_out = list(map(str.strip, " ".join(args).split(" > "))) + if len(redirect_out) > 1: + args, filepath = redirect_out[0].split(), redirect_out[-1] + kwargs.update(stdout=open(filepath, "w")) + return subprocess.call(args, **kwargs)