From 5007d5292fc0f2e4ba2b8377302bcddc4330a003 Mon Sep 17 00:00:00 2001 From: Nir Izraeli Date: Thu, 1 Nov 2018 17:51:53 -0700 Subject: [PATCH 1/3] Move setup to release directory --- setup.py => release/setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename setup.py => release/setup.py (91%) diff --git a/setup.py b/release/setup.py similarity index 91% rename from setup.py rename to release/setup.py index 61bd451d7..45b09515f 100755 --- a/setup.py +++ b/release/setup.py @@ -31,8 +31,7 @@ def build_setup(package_base, package_name, version_path, package_data = {} # generate install_requires based on requirements.txt - base_path = os.path.abspath(os.path.dirname(__file__)) - requirements_path = os.path.join(base_path, package_base, "requirements.txt") + requirements_path = os.path.join(package_base, "requirements.txt") if os.path.exists(requirements_path): install_requires = get_requirements(requirements_path) # include requirementst.txt as part of package @@ -42,14 +41,14 @@ def build_setup(package_base, package_name, version_path, else: install_requires = [] - test_requirements_path = os.path.join(base_path, "tests", package_base, + test_requirements_path = os.path.join("tests", package_base, "requirements.txt") extras_require = {} if os.path.exists(test_requirements_path): extras_require['test'] = get_requirements(test_requirements_path) - version_path = os.path.join(base_path, package_base, version_path) - readme_path = os.path.join(base_path, "README.rst") + version_path = os.path.join(package_base, version_path) + readme_path = "README.rst" setup( script_args=script_args, name=package_name, @@ -90,6 +89,7 @@ def build_setup_idaplugin(script_args=None): if __name__ == '__main__': + print(os.getcwd()) expected_packages = {'server', 'idaplugin'} packages = set(os.listdir('.')) & expected_packages From f3f05bb842d7d1a67d801ab2ac9c8b4d6724d1f3 Mon Sep 17 00:00:00 2001 From: Nir Izraeli Date: Thu, 1 Nov 2018 20:40:02 -0700 Subject: [PATCH 2/3] Python based release tool --- .gitignore | 1 + release/__init__.py | 0 release/package.py | 89 +++++++++++++++++++++++++++++++++ release/packages.py | 21 ++++++++ release/release.py | 73 ++++++++++++++++++++++++++++ release/setup.py | 116 -------------------------------------------- 6 files changed, 184 insertions(+), 116 deletions(-) create mode 100644 release/__init__.py create mode 100644 release/package.py create mode 100644 release/packages.py create mode 100644 release/release.py delete mode 100755 release/setup.py diff --git a/.gitignore b/.gitignore index ddc5d4274..b715d85d8 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ Thumbs.db # server private key file server/rematch/.rematch_secret.key server/postgres-data/ +release/dist/ diff --git a/release/__init__.py b/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/release/package.py b/release/package.py new file mode 100644 index 000000000..9d3dcf426 --- /dev/null +++ b/release/package.py @@ -0,0 +1,89 @@ +import os +import re + +from pkg_resources import parse_version +from setuptools import setup, find_packages +from twine.cli import dispatch as twine + + +class Package(object): + def __init__(self, name, path, version_path, zip_safe, package_data=None, + classifiers=[]): + self.name = name + self.path = path + self.version_path = os.path.join(self.path, version_path, "version.py") + self.zip_safe = zip_safe + self.classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)" + ] + self.classifiers += classifiers + self.package_data = package_data or {} + + def get_version(self): + version_content = open(self.version_path).read() + # grab version string from file, this excludes inline comments too + version_str = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + version_content).group(1) + return parse_version(version_str) + + def get_released_version(self): + # TODO + pass + + def generate_changelog(self): + # TODO + pass + + def get_requirements(self, *parts): + fpath = os.path.join(*parts) + if not os.path.exists(fpath): + return [] + + with open(fpath) as fh: + return (l for l in fh.readlines() if not l.startswith('-r ')) + + def build(self, *script_args): + # generate install_requires based on requirements.txt + install_requires = self.get_requirements(self.path, "requirements.txt") + + # include requirementst.txt as part of package + if install_requires: + if self.path not in self.package_data: + self.package_data[self.path] = [] + self.package_data[self.path].append('requirements.txt') + + extras_require = {'test': self.get_requirements("tests", self.path, + "requirements.txt")} + + with open("README.rst") as fh: + long_description = fh.read() + + setup( + script_args=script_args + ('--dist-dir=./release/dist',), + name=self.name, + version=str(self.get_version()), + author="Nir Izraeli", + author_email="nirizr@gmail.com", + description=("A IDA Pro plugin and server framework for binary " + "function level diffing."), + keywords=["rematch", "ida", "idapro", "binary diffing", + "reverse engineering"], + url="https://www.github.com/nirizr/rematch/", + packages=find_packages(self.path), + package_dir={'': self.path}, + package_data=self.package_data, + extras_require=extras_require, + install_requires=install_requires, + long_description=long_description, + classifiers=self.classifiers + ) + + def get_dist_file(self): + return './release/dist/{}-{}.zip'.format(self.name, self.get_version()) + + def upload(self, repo="pypi"): + twine(['upload', self.get_dist_file(), '-r', repo]) + + def __repr__(self): + return "".format(self.name, self.get_version()) diff --git a/release/packages.py b/release/packages.py new file mode 100644 index 000000000..4be2366ec --- /dev/null +++ b/release/packages.py @@ -0,0 +1,21 @@ +from .package import Package + + +server = Package(name='rematch-server', path='server', version_path='./', + classifiers=["Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Environment :: Web Environment", + "Framework :: Django"], zip_safe=True) + +ida = Package(name='rematch-idaplugin', path='idaplugin', + version_path='rematch', zip_safe=False, + package_data={'idaplugin/rematch': ['images/*']}, + classifiers=["Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7"]) + +package_list = [server, ida] diff --git a/release/release.py b/release/release.py new file mode 100644 index 000000000..b99e86ee6 --- /dev/null +++ b/release/release.py @@ -0,0 +1,73 @@ +#!python + +import logging +import subprocess +import argparse + +from . import setup +from .packages import package_list + + +REMOTE="origin" +BRANCH="master" +# TODO: Remove next line just before merging +BRANCH="nirizr/release_py" + + +def sh_exec(*args): + logging.getLogger('sh_exec').info("Executing '%s'", args) + output = subprocess.check_output(args, shell=False).strip().decode() + logging.getLogger('sh_exec').info("Output '%s'", output) + return output + + +def validate_git_state(): + logging.info("Validating git state is clean") + + if sh_exec("git", "rev-parse", "--abbrev-ref", "HEAD") != BRANCH: + raise RuntimeError("Current branch name doesn't match release branch.") + + if "nothing to commit" not in sh_exec("git", "status", "-uno"): + raise RuntimeError("Local branch is dirty, can only release in clean " + "workspaces.") + + remote_branch = sh_exec("git", "ls-remote", REMOTE, "-h", + "refs/heads/" + BRANCH) + remote_branch_hash = remote_branch.split()[0] + if sh_exec("git", "rev-parse", BRANCH) is not remote_branch_hash: + raise RuntimeError("Local and remote branches are out of sync, releases " + "are only possible on up-to-date branch") + + +def identify_new_packages(): + new_packages = set() + + for package in package_list: + print(package) + # if package.get_version() + new_packages.add(package) + + return new_packages + + +def main(): + parser = argparse.ArgumentParser(description="Rematch release utility") + parser.add_argument('--verbose', '-v', action='count') + parser.add_argument('--skip-validation', '-sv', default=False, action='store_true') + args = parser.parse_args() + + logging.basicConfig(level=logging.ERROR - args.verbose * 10) + + if not args.skip_validation: + validate_git_state() + + packages = identify_new_packages() + for package in packages: + package.generate_changelog() + package.build('sdist', '--formats=zip') + package.upload('test') + package.upload() + + +if __name__ == '__main__': + main() diff --git a/release/setup.py b/release/setup.py deleted file mode 100755 index 45b09515f..000000000 --- a/release/setup.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/python - -import sys -import os -from setuptools import setup, find_packages -import re - - -# Utility function to read the README file. -# Used for the long_description. It's nice, because now 1) we have a top level -# README file and 2) it's easier to type in the README file than to put a raw -# string in below ... -def read(fname): - return open(fname).read() - - -def get_version(path): - version_path = os.path.join(path, 'version.py') - return re.search( - r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', # It excludes inline comment too - open(version_path).read()).group(1) - - -def get_requirements(fname): - return (l for l in open(fname).readlines() if not l.startswith('-r ')) - - -def build_setup(package_base, package_name, version_path, - package_data=None, script_args=None): - if package_data is None: - package_data = {} - - # generate install_requires based on requirements.txt - requirements_path = os.path.join(package_base, "requirements.txt") - if os.path.exists(requirements_path): - install_requires = get_requirements(requirements_path) - # include requirementst.txt as part of package - if package_base not in package_data: - package_data[package_base] = [] - package_data[package_base].append('requirements.txt') - else: - install_requires = [] - - test_requirements_path = os.path.join("tests", package_base, - "requirements.txt") - extras_require = {} - if os.path.exists(test_requirements_path): - extras_require['test'] = get_requirements(test_requirements_path) - - version_path = os.path.join(package_base, version_path) - readme_path = "README.rst" - setup( - script_args=script_args, - name=package_name, - version=get_version(version_path), - author="Nir Izraeli", - author_email="nirizr@gmail.com", - description=("A IDA Pro plugin and server framework for binary function " - "level diffing."), - keywords=["rematch", "ida", "idapro", "bindiff", "binary diffing", - "reverse engineering"], - url="https://www.github.com/nirizr/rematch/", - packages=find_packages(package_base), - package_dir={'': package_base}, - package_data=package_data, - extras_require=extras_require, - install_requires=install_requires, - long_description=read(readme_path), - classifiers=[ - "Development Status :: 3 - Alpha", - ], - ) - - -def build_setup_server(script_args=None): - build_setup(package_base='server', - package_name='rematch-server', - version_path='./', - script_args=script_args) - - -def build_setup_idaplugin(script_args=None): - package_data = {'idaplugin/rematch': ['images/*']} - build_setup(package_base='idaplugin', - package_name='rematch-idaplugin', - version_path='rematch', - package_data=package_data, - script_args=script_args) - - -if __name__ == '__main__': - print(os.getcwd()) - expected_packages = {'server', 'idaplugin'} - packages = set(os.listdir('.')) & expected_packages - - if len(sys.argv) < 2 and len(packages) > 1: - print("Usage: {} {{package name}}".format(sys.argv[0])) - print("Available packages are: {}".format(", ".join(packages))) - sys.exit(1) - - # If all packages are available, allow a 'release' command that would push - # all packages to pypi - if sys.argv[1] == 'release' and packages == expected_packages: - script_args = ['sdist', '--dist-dir=./dist', '--formats=zip', 'upload'] - if not (len(sys.argv) >= 3 and sys.argv[2] == 'official'): - script_args += ['-r', 'pypitest'] - build_setup_server(script_args=script_args) - build_setup_idaplugin(script_args=script_args) - else: - package = packages.pop() if len(packages) == 1 else sys.argv[1] - if sys.argv[1] == package: - sys.argv = sys.argv[:1] + sys.argv[2:] - if package == 'server': - build_setup_server() - elif package == 'idaplugin': - build_setup_idaplugin() From d94d44e56cb167f2b1dbb140d7f3a74ef1846b62 Mon Sep 17 00:00:00 2001 From: Nir Izraeli Date: Fri, 2 Nov 2018 02:11:00 -0700 Subject: [PATCH 3/3] replace setup with release in travis checks --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba22b391d..79ba79ea7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ matrix: services: - docker - python: "2.7" - env: PROJECT=setup.py + env: PROJECT=release - python: "3.6" - env: PROJECT=setup.py + env: PROJECT=release - python: "3.6" env: PROJECT=docs - python: "2.7" @@ -98,8 +98,8 @@ script: py.test -rapP ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=${PROJECT} ; fi ; fi ; - - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py server install ; fi ; - - if [ "${PROJECT}" == "setup.py" ]; then python ./setup.py idaplugin install ; fi ; + - if [ "${PROJECT}" == "release" ]; then python ./release server install ; fi ; + - if [ "${PROJECT}" == "release" ]; then python ./release idaplugin install ; fi ; - if [ "${PROJECT}" == "docs" ]; then sphinx-build ./docs/ ./docs/_build -a -E -n -W ; fi ; after_script: