diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index b2f7b6b..002cf64 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -16,8 +16,11 @@ jobs: with: python-version: ${{ matrix.python_version }} - name: Install requirements - run: pip install -e . -r tests/requirements.txt -r unittests/requirements.txt pylint - - name: Check with pylint - run: pylint scottypy tests unittests + run: pip install -e . -r unittests/requirements.txt pylint mypy + - name: Format + if: matrix.python_version == '3.8' + run: pip install black isort && make format + - name: Lint + run: make lint - name: Unittest - run: pytest unittests + run: make test diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml new file mode 100644 index 0000000..d5f3859 --- /dev/null +++ b/.github/workflows/pythonpublish.yml @@ -0,0 +1,31 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.pylintrc b/.pylintrc index 7f67c80..9fb6558 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [MESSAGES CONTROL] -disable=R,broad-except,protected-access,unused-argument,redefined-builtin,missing-docstring,invalid-name,ungrouped-imports,wrong-import-order +disable=R,broad-except,protected-access,unused-argument,redefined-builtin,missing-docstring,invalid-name,ungrouped-imports,wrong-import-order,bad-continuation [FORMAT] max-line-length=150 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..77128ac --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +default: check + + +format: + isort -c -rc scottypy unittests + black --check scottypy unittests + +do_format: + isort -rc scottypy unittests + black scottypy unittests + +test: + pytest unittests/ + +lint: + MYPYPATH=stubs mypy scottypy --strict + pylint --rcfile .pylintrc -j $(shell nproc) scottypy unittests + +check: format lint test diff --git a/README.md b/README.md index 7bff399..0102622 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ git push origin : --tags ## ChangeLog +### 0.23.1 + +* Provide more information on exceptions +* Add type hinting +* Fix bug with downloading windows paths on linux + ### 0.22.1 * Fix bug with prefetch and then beam up without explicitly setting version diff --git a/scottypy/__init__.py b/scottypy/__init__.py index dbb5d79..2beecfd 100644 --- a/scottypy/__init__.py +++ b/scottypy/__init__.py @@ -1,4 +1,4 @@ -from .scotty import Scotty -from .file import File from .beam import Beam from .exc import NotOverwriting +from .file import File +from .scotty import Scotty diff --git a/scottypy/__version__.py b/scottypy/__version__.py index d74a474..43e16f5 100644 --- a/scottypy/__version__.py +++ b/scottypy/__version__.py @@ -1 +1 @@ -__version__ = "0.22.1" +__version__ = "0.23.1" diff --git a/scottypy/app.py b/scottypy/app.py index 1dab42b..971d235 100644 --- a/scottypy/app.py +++ b/scottypy/app.py @@ -1,56 +1,75 @@ from __future__ import print_function + +import json import logging -import re import os -import json +import re +import typing import webbrowser -import capacity from getpass import getpass + +import capacity import click -from . import Scotty, NotOverwriting + +from .exc import NotOverwriting +from .scotty import Scotty +from .types import JSON + +if typing.TYPE_CHECKING: + from .beam import Beam _CONFIG_PATH = os.path.expanduser("~/.scotty.conf") _BEAM_PATH = re.compile(r"^([^@:]+)@([^@:]+):(.*)$") -def _get_config(): +def _get_config() -> JSON: try: with open(_CONFIG_PATH, "r") as f: - return json.load(f) + config = json.load(f) # type: JSON + return config except Exception: return {} -def _save_config(config): +def _save_config(config: JSON) -> None: with open(_CONFIG_PATH, "w") as f: json.dump(config, f) @click.group() -def main(): +def main() -> None: pass -def _get_url(): +def _get_url() -> str: config = _get_config() - if not config or 'url' not in config: + if not config or "url" not in config: raise click.ClickException( """The URL of Scotty has not been set. -You can set the URL either by using the --url flag or by running \"scotty set_url http://some.scotty.com\"""") - return config['url'] +You can set the URL either by using the --url flag or by running \"scotty set_url http://some.scotty.com\"""" + ) + url = config["url"] # type: str + return url -def _write_beam_info(beam, directory): - with open(os.path.join(directory, 'beam.txt'), 'w') as f: - f.write("""Start: {start} +def _write_beam_info(beam: "Beam", directory: str) -> None: + with open(os.path.join(directory, "beam.txt"), "w") as f: + f.write( + """Start: {start} Host: {host} Directory: {directory} Comment: {comment} -""".format(start=beam.start, host=beam.host, directory=beam.directory, comment=beam.comment)) +""".format( + start=beam.start, + host=beam.host, + directory=beam.directory, + comment=beam.comment, + ) + ) -def _link_beam(storage_base, beam, dest): +def _link_beam(storage_base: str, beam: "Beam", dest: str) -> None: if not os.path.isdir(dest): os.makedirs(dest) @@ -64,16 +83,18 @@ def _link_beam(storage_base, beam, dest): @main.command() @click.argument("beam_id_or_tag") -@click.option('--url', default=_get_url, help='Base URL of Scotty') -@click.option('--storage_base', default='/var/scotty', help='Base location of Scotty\'s storage') -@click.option('-d', '--dest', default=None, help='Link destination') -def link(beam_id_or_tag, url, storage_base, dest): +@click.option("--url", default=_get_url, help="Base URL of Scotty") +@click.option( + "--storage_base", default="/var/scotty", help="Base location of Scotty's storage" +) +@click.option("-d", "--dest", default=None, help="Link destination") +def link(beam_id_or_tag: str, url: str, storage_base: str, dest: str) -> None: """Create symbolic links representing a single beam or a set of beams by their tag ID. To link a specific beam just use write its id as an argument. To link an entire tag specify t:[tag_name] as an argument, replacing [tag_name] with the name of the tag""" scotty = Scotty(url) - if beam_id_or_tag.startswith('t:'): + if beam_id_or_tag.startswith("t:"): tag = beam_id_or_tag[2:] if dest is None: dest = tag @@ -90,12 +111,12 @@ def link(beam_id_or_tag, url, storage_base, dest): @main.command() @click.argument("beam_id_or_tag") -@click.option('--url', default=_get_url, help='Base URL of Scotty') -def show(beam_id_or_tag, url): +@click.option("--url", default=_get_url, help="Base URL of Scotty") +def show(beam_id_or_tag: str, url: str) -> None: """List the files of the given beam or tag""" scotty = Scotty(url) - def _list(beam): + def _list(beam: "Beam") -> None: print("Beam #{}".format(beam.id)) print(" Host: {}".format(beam.host)) print(" Directory: {}".format(beam.directory)) @@ -106,7 +127,7 @@ def _list(beam): print("") - if beam_id_or_tag.startswith('t:'): + if beam_id_or_tag.startswith("t:"): tag = beam_id_or_tag[2:] for beam in scotty.get_beams_by_tag(tag): _list(beam) @@ -114,7 +135,7 @@ def _list(beam): _list(scotty.get_beam(beam_id_or_tag)) -def _download_beam(beam, dest, overwrite, filter): +def _download_beam(beam: "Beam", dest: str, overwrite: bool, filter: str) -> None: if not os.path.isdir(dest): os.makedirs(dest) @@ -134,17 +155,28 @@ def _download_beam(beam, dest, overwrite, filter): @main.command() @click.argument("beam_id_or_tag") -@click.option('--dest', default=None, help='Destination directory') -@click.option('--url', default=_get_url, help='Base URL of Scotty') -@click.option('-f', '--filter', default=None, help="Download only files that contain the given string in their name (case insensetive)") -@click.option('--overwrite/--no-overwrite', default=False, help='Overwrite existing files on the disk') -def down(beam_id_or_tag, dest, url, overwrite, filter): # pylint: disable=W0622 +@click.option("--dest", default=None, help="Destination directory") +@click.option("--url", default=_get_url, help="Base URL of Scotty") +@click.option( + "-f", + "--filter", + default=None, + help="Download only files that contain the given string in their name (case insensetive)", +) +@click.option( + "--overwrite/--no-overwrite", + default=False, + help="Overwrite existing files on the disk", +) +def down( + beam_id_or_tag: str, dest: str, url: str, overwrite: bool, filter: str +) -> None: # pylint: disable=W0622 """Download a single beam or a set of beams by their tag ID. To download a specific beam just use write its id as an argument. To download an entire tag specify t:[tag_name] as an argument, replacing [tag_name] with the name of the tag""" scotty = Scotty(url) - if beam_id_or_tag.startswith('t:'): + if beam_id_or_tag.startswith("t:"): tag = beam_id_or_tag[2:] if dest is None: dest = tag @@ -159,40 +191,64 @@ def down(beam_id_or_tag, dest, url, overwrite, filter): # pylint: disable=W0622 @main.group() -def up(): +def up() -> None: pass @up.command() @click.argument("directory") -@click.option('--url', default=_get_url, help='Base URL of Scotty') -@click.option('-t', '--tag', 'tags', multiple=True, - help='Tag to be associated with the beam. Can be specified multiple times') -def local(directory, url, tags): - logging.basicConfig(format='%(name)s:%(levelname)s:%(message)s', level=logging.DEBUG) +@click.option("--url", default=_get_url, help="Base URL of Scotty") +@click.option( + "-t", + "--tag", + "tags", + multiple=True, + help="Tag to be associated with the beam. Can be specified multiple times", +) +def local(directory: str, url: str, tags: typing.List[str]) -> None: + logging.basicConfig( + format="%(name)s:%(levelname)s:%(message)s", level=logging.DEBUG + ) scotty = Scotty(url) - click.echo('Beaming up {}'.format(directory)) + click.echo("Beaming up {}".format(directory)) beam_id = scotty.beam_up(directory, tags=tags) - click.echo('Successfully beamed beam #{}'.format(beam_id)) + click.echo("Successfully beamed beam #{}".format(beam_id)) @up.command() @click.argument("path") @click.option("--rsa_key", type=click.Path(exists=True, dir_okay=False)) @click.option("--email") -@click.option("--goto", is_flag=True, default=False, help="Open your browser at the beam page") -@click.option('--url', default=_get_url, help='Base URL of Scotty') +@click.option( + "--goto", is_flag=True, default=False, help="Open your browser at the beam page" +) +@click.option("--url", default=_get_url, help="Base URL of Scotty") @click.option("--stored_key", default=None) -@click.option('-t', '--tag', 'tags', multiple=True, - help='Tag to be associated with the beam. Can be specified multiple times') -def remote(url, path, rsa_key, email, goto, stored_key, tags): +@click.option( + "-t", + "--tag", + "tags", + multiple=True, + help="Tag to be associated with the beam. Can be specified multiple times", +) +def remote( + url: str, + path: str, + rsa_key: str, + email: str, + goto: bool, + stored_key: str, + tags: typing.List[str], +) -> None: scotty = Scotty(url) m = _BEAM_PATH.search(path) if not m: - raise click.ClickException("Invalid path. Path should be in the form of user@host:/path/to/directory") + raise click.ClickException( + "Invalid path. Path should be in the form of user@host:/path/to/directory" + ) password = None user, host, directory = m.groups() @@ -203,9 +259,21 @@ def remote(url, path, rsa_key, email, goto, stored_key, tags): pass else: password = getpass("Password for {}@{}: ".format(user, host)) - beam_id = scotty.initiate_beam(user, host, directory, password, rsa_key, email, stored_key=stored_key, tags=tags) - click.echo("Successfully initiated beam #{} to {}@{}:{}".format( - beam_id, user, host, directory)) + beam_id = scotty.initiate_beam( + user, + host, + directory, + password, + rsa_key, + email, + stored_key=stored_key, + tags=tags, + ) + click.echo( + "Successfully initiated beam #{} to {}@{}:{}".format( + beam_id, user, host, directory + ) + ) beam_url = "{}/#/beams/{}".format(url, beam_id) if goto: @@ -215,11 +283,13 @@ def remote(url, path, rsa_key, email, goto, stored_key, tags): @main.command("tag") -@click.option("-d", "--delete", help="Delete the specified tag", is_flag=True, default=False) -@click.option('--url', default=_get_url, help='Base URL of Scotty') +@click.option( + "-d", "--delete", help="Delete the specified tag", is_flag=True, default=False +) +@click.option("--url", default=_get_url, help="Base URL of Scotty") @click.argument("tag") @click.argument("beam") -def tag_beam(tag, beam, delete, url): +def tag_beam(tag: str, beam: int, delete: bool, url: str) -> None: scotty = Scotty(url) if delete: scotty.remove_tag(beam, tag) @@ -229,21 +299,21 @@ def tag_beam(tag, beam, delete, url): @main.command() @click.argument("url") -def set_url(url): +def set_url(url: str) -> None: config = _get_config() scotty = Scotty(url) scotty.sanity_check() - config['url'] = url + config["url"] = url _save_config(config) @main.command() @click.argument("beam_id") @click.argument("comment") -@click.option('--url', default=_get_url, help='Base URL of Scotty') -def set_comment(beam_id, url, comment): +@click.option("--url", default=_get_url, help="Base URL of Scotty") +def set_comment(beam_id: int, url: str, comment: str) -> None: """Set a comment for the specified beam""" scotty = Scotty(url) diff --git a/scottypy/beam.py b/scottypy/beam.py index 93ed3c2..09afa03 100644 --- a/scottypy/beam.py +++ b/scottypy/beam.py @@ -1,7 +1,18 @@ -import dateutil.parser import json +import typing + +import dateutil.parser from pact import Pact +from scottypy.utils import raise_for_status + +from .types import JSON + +if typing.TYPE_CHECKING: + from datetime import datetime + from .scotty import Scotty + from .file import File + class Beam(object): """A class representing a single beam. @@ -18,8 +29,25 @@ class Beam(object): :ivar purge_time: The number of days left for this beam to exist if it has no pinners. :ivar size: The total size of the beam in bytes. """ - def __init__(self, scotty, id_, file_ids, initiator_id, start, deleted, completed, pins, host, error, directory, - purge_time, size, comment, associated_issues): + + def __init__( + self, + scotty: "Scotty", + id_: int, + file_ids: typing.List[int], + initiator_id: int, + start: "datetime", + deleted: bool, + completed: bool, + pins: typing.List[int], + host: str, + error: str, + directory: str, + purge_time: int, + size: int, + comment: str, + associated_issues: typing.List[int], + ): self.id = id_ self._file_ids = file_ids self.initiator_id = initiator_id @@ -37,75 +65,81 @@ def __init__(self, scotty, id_, file_ids, initiator_id, start, deleted, complete self._comment = comment @property - def comment(self): + def comment(self) -> str: return self._comment - def update(self): + def update(self) -> None: """Update the status of the beam object""" - response = self._scotty.session.get("{0}/beams/{1}".format(self._scotty.url, self.id)) - response.raise_for_status() - beam_obj = response.json()['beam'] - - self._file_ids = beam_obj['files'] - self.deleted = beam_obj['deleted'] - self.completed = beam_obj['completed'] - self.pins = beam_obj['pins'] - self.error = beam_obj['error'] - self.purge_time = beam_obj['purge_time'] - self.associated_issues = beam_obj['associated_issues'] - self.size = beam_obj['size'] - self._comment = beam_obj['comment'] + response = self._scotty.session.get( + "{0}/beams/{1}".format(self._scotty.url, self.id) + ) + raise_for_status(response) + beam_obj = response.json()["beam"] + + self._file_ids = beam_obj["files"] + self.deleted = beam_obj["deleted"] + self.completed = beam_obj["completed"] + self.pins = beam_obj["pins"] + self.error = beam_obj["error"] + self.purge_time = beam_obj["purge_time"] + self.associated_issues = beam_obj["associated_issues"] + self.size = beam_obj["size"] + self._comment = beam_obj["comment"] @classmethod - def from_json(cls, scotty, json_node): + def from_json(cls, scotty: "Scotty", json_node: JSON) -> "Beam": return cls( scotty, - json_node['id'], - json_node.get('files', []), - json_node['initiator'], - dateutil.parser.parse(json_node['start']), - json_node['deleted'], - json_node['completed'], - json_node['pins'], - json_node['host'], - json_node['error'], - json_node['directory'], - json_node['purge_time'], - json_node['size'], - json_node['comment'], - json_node['associated_issues']) - - def iter_files(self): + json_node["id"], + json_node.get("files", []), + json_node["initiator"], + dateutil.parser.parse(json_node["start"]), + json_node["deleted"], + json_node["completed"], + json_node["pins"], + json_node["host"], + json_node["error"], + json_node["directory"], + json_node["purge_time"], + json_node["size"], + json_node["comment"], + json_node["associated_issues"], + ) + + def iter_files(self) -> typing.Iterator["File"]: """Iterate the beam files one by one, yielding :class:`.File` objects This function might be slow when used with beams containing large number of file. Consider using :func:`.get_files` instead.""" for id_ in self._file_ids: yield self._scotty.get_file(id_) - def get_files(self, filter_=None): + def get_files(self, filter_: typing.Optional[str] = None) -> typing.List["File"]: """Get a list of :class:`.File` instances representing the beam files. :ivar filter_: Optional filter string. When given, only files which their name contains the filter will be returned.""" return self._scotty.get_files(self.id, filter_) - def set_comment(self, comment): - data = {'beam': {'comment': comment}} + def set_comment(self, comment: str) -> None: + data = {"beam": {"comment": comment}} response = self._scotty.session.put( - "{0}/beams/{1}".format(self._scotty.url, self.id), - data=json.dumps(data)) - response.raise_for_status() + "{0}/beams/{1}".format(self._scotty.url, self.id), data=json.dumps(data) + ) + raise_for_status(response) self._comment = comment - def set_issue_association(self, issue_id, associated): - self._scotty.session.request( - 'POST' if associated else 'DELETE', - "{0}/beams/{1}/issues/{2}".format(self._scotty.url, self.id, issue_id)).raise_for_status() + def set_issue_association(self, issue_id: str, associated: bool) -> None: + raise_for_status( + self._scotty.session.request( + "POST" if associated else "DELETE", + "{0}/beams/{1}/issues/{2}".format(self._scotty.url, self.id, issue_id), + ) + ) - def _check_finish(self): + def _check_finish(self) -> bool: self.update() return self.completed - def get_pact(self): + def get_pact(self) -> Pact: """Get a Pact instance. The pact is finished when the beam has been completed""" pact = Pact("Waiting for beam {}".format(self.id)) pact.until(self._check_finish) diff --git a/scottypy/exc.py b/scottypy/exc.py index b80001e..86fdc77 100644 --- a/scottypy/exc.py +++ b/scottypy/exc.py @@ -1,9 +1,9 @@ class PathNotExists(Exception): - def __init__(self, path): + def __init__(self, path: str): super(PathNotExists, self).__init__("{} does not exist".format(path)) class NotOverwriting(Exception): - def __init__(self, file_): + def __init__(self, file_: str): super(NotOverwriting, self).__init__() self.file = file_ diff --git a/scottypy/file.py b/scottypy/file.py index ad5d044..f5f56fa 100644 --- a/scottypy/file.py +++ b/scottypy/file.py @@ -1,13 +1,22 @@ import os -import dateutil.parser +import typing from datetime import datetime + +import dateutil.parser + from .exc import NotOverwriting +from .types import JSON +from .utils import fix_path_sep_for_current_platform, raise_for_status + +if typing.TYPE_CHECKING: + from requests import Session + _CHUNK_SIZE = 1024 ** 2 * 4 _EPOCH = datetime.utcfromtimestamp(0) -def _to_epoch(d): +def _to_epoch(d: datetime) -> float: return (d - _EPOCH.replace(tzinfo=d.tzinfo)).total_seconds() @@ -21,7 +30,17 @@ class File(object): :ivar size: The size of the file in bytes. :ivar url: A URL for downloading the file.""" - def __init__(self, session, id_, file_name, status, storage_name, size, url, mtime): + def __init__( + self, + session: "Session", + id_: int, + file_name: str, + status: str, + storage_name: str, + size: int, + url: str, + mtime: typing.Optional[datetime], + ): self.id = id_ self._session = session @@ -33,23 +52,31 @@ def __init__(self, session, id_, file_name, status, storage_name, size, url, mti self.mtime = mtime @classmethod - def from_json(cls, session, json_node): + def from_json(cls, session: "Session", json_node: JSON) -> "File": raw_mtime = json_node.get("mtime") mtime = None if raw_mtime is None else dateutil.parser.parse(raw_mtime) - return cls(session, json_node['id'], json_node['file_name'], json_node['status'], json_node['storage_name'], - json_node['size'], json_node['url'], mtime) - - def stream_to(self, fileobj): + return cls( + session, + json_node["id"], + json_node["file_name"], + json_node["status"], + json_node["storage_name"], + json_node["size"], + json_node["url"], + mtime, + ) + + def stream_to(self, fileobj: "typing.BinaryIO") -> None: """Fetch the file content from the server and write it to fileobj""" response = self._session.get(self.url, stream=True) - response.raise_for_status() + raise_for_status(response) for chunk in response.iter_content(chunk_size=_CHUNK_SIZE): fileobj.write(chunk) - def download(self, directory=".", overwrite=False): + def download(self, directory: str = ".", overwrite: bool = False) -> None: """Download the file to the specified directory, retaining its name""" - subdir, file_ = os.path.split(self.file_name) + subdir, file_ = os.path.split(fix_path_sep_for_current_platform(self.file_name)) subdir = os.path.join(directory, subdir) file_ = os.path.join(subdir, file_) @@ -69,7 +96,7 @@ def download(self, directory=".", overwrite=False): mtime = _to_epoch(self.mtime) os.utime(file_, (mtime, mtime)) - def link(self, storage_base, dest): + def link(self, storage_base: str, dest: str) -> None: source_path = os.path.join(storage_base, self.storage_name) link_path = os.path.join(dest, self.file_name) link_dir = os.path.split(link_path)[0] diff --git a/scottypy/scotty.py b/scottypy/scotty.py index acba896..55ee33c 100644 --- a/scottypy/scotty.py +++ b/scottypy/scotty.py @@ -1,3 +1,4 @@ +import abc import errno import json import logging @@ -7,6 +8,8 @@ import subprocess import sys import tempfile +import types +import typing from tempfile import NamedTemporaryFile from uuid import uuid4 @@ -18,6 +21,8 @@ from .beam import Beam from .exc import PathNotExists from .file import File +from .types import JSON +from .utils import raise_for_status _SLEEP_TIME = 10 _NUM_OF_RETRIES = (60 // _SLEEP_TIME) * 15 @@ -26,62 +31,93 @@ logger = logging.getLogger("scotty") # type: logging.Logger -class CombadgePython: - version = "v1" +class Combadge: + @property + @abc.abstractmethod + def version(self) -> str: + pass + + @classmethod + @abc.abstractmethod + def from_response(cls, response: requests.Response) -> "Combadge": + pass + + @abc.abstractmethod + def remove(self) -> None: + pass + + @abc.abstractmethod + def run(self, *, beam_id: int, directory: str, transporter_host: str) -> None: + pass + - def __init__(self, combadge_module): +class CombadgePython(Combadge): + version = "v1" # type: str + + def __init__(self, combadge_module: types.ModuleType): self._combadge_module = combadge_module @classmethod - def from_response(cls, response): - with NamedTemporaryFile(mode="w", suffix='.py', delete=False) as combadge_file: + def from_response(cls, response: requests.Response) -> "CombadgePython": + with NamedTemporaryFile(mode="w", suffix=".py", delete=False) as combadge_file: combadge_file.write(response.text) combadge_file.flush() return cls(emport.import_file(combadge_file.name)) - def remove(self): + def remove(self) -> None: os.remove(self._combadge_module.__file__) - def run(self, *, beam_id, directory, transporter_host): - self._combadge_module.beam_up(beam_id, directory, transporter_host) + def run(self, *, beam_id: int, directory: str, transporter_host: str) -> None: + self._combadge_module.beam_up(beam_id, directory, transporter_host) # type: ignore -class CombadgeRust: - version = "v2" +class CombadgeRust(Combadge): + version = "v2" # type: str - def __init__(self, file_name): + def __init__(self, file_name: str): self._file_name = file_name @classmethod - def _generate_random_combadge_name(cls, string_length): + def _generate_random_combadge_name(cls, string_length: int) -> str: random_string = str(uuid4())[:string_length] return "combadge_{random_string}".format(random_string=random_string) @classmethod - def _get_local_combadge_path(cls): + def _get_local_combadge_path(cls) -> str: combadge_name = cls._generate_random_combadge_name(string_length=10) local_combadge_dir = tempfile.gettempdir() return os.path.join(local_combadge_dir, combadge_name) @classmethod - def from_response(cls, response): + def from_response(cls, response: requests.Response) -> "CombadgeRust": local_combadge_path = cls._get_local_combadge_path() - with open(local_combadge_path, 'wb') as combadge_file: + with open(local_combadge_path, "wb") as combadge_file: for chunk in response.iter_content(chunk_size=1024): combadge_file.write(chunk) st = os.stat(local_combadge_path) os.chmod(local_combadge_path, st.st_mode | stat.S_IEXEC) return cls(combadge_file.name) - def remove(self): + def remove(self) -> None: try: os.remove(self._file_name) except OSError as e: if e.errno != errno.ENOENT: raise - def run(self, *, beam_id, directory, transporter_host): - subprocess.run([self._file_name, '-b', str(beam_id), '-p', directory, '-t', transporter_host], check=False) + def run(self, *, beam_id: int, directory: str, transporter_host: str) -> None: + subprocess.run( + [ + self._file_name, + "-b", + str(beam_id), + "-p", + directory, + "-t", + transporter_host, + ], + check=False, + ) class Scotty(object): @@ -89,57 +125,76 @@ class Scotty(object): :param str url: The base URL of Scotty.""" - def __init__(self, url, retry_times=3, backoff_factor=2): + def __init__(self, url: str, retry_times: int = 3, backoff_factor: int = 2): self._url = url self._session = requests.Session() - self._session.headers.update({ - 'Accept-Encoding': 'gzip', - 'Content-Type': 'application/json'}) + self._session.headers.update( + {"Accept-Encoding": "gzip", "Content-Type": "application/json"} + ) self._session.mount( - url, HTTPAdapter( - max_retries=Retry(total=retry_times, status_forcelist=[502, 504], backoff_factor=backoff_factor))) - self._combadge = None + url, + HTTPAdapter( + max_retries=Retry( + total=retry_times, + status_forcelist=[502, 504], + backoff_factor=backoff_factor, + ) + ), + ) + self._combadge = None # type: typing.Optional[Combadge] - def prefetch_combadge(self, combadge_version=_DEFAULT_COMBADGE_VERSION): + def prefetch_combadge( + self, combadge_version: str = _DEFAULT_COMBADGE_VERSION + ) -> None: """Prefetch the combadge to a temporary file. Future beams will use that combadge instead of having to re-download it.""" self._get_combadge(combadge_version=combadge_version) - def remove_combadge(self): - self._combadge.remove() + def remove_combadge(self) -> None: + if self._combadge: + self._combadge.remove() - def _get_combadge(self, combadge_version): + def _get_combadge(self, combadge_version: str) -> "Combadge": """Get the combadge from the memory if it has been prefetched. Otherwise, download it from Scotty""" if self._combadge and self._combadge.version == combadge_version: return self._combadge - response = self._session.get("{}/combadge".format(self._url), timeout=_TIMEOUT, params={ - "combadge_version": combadge_version, - "os_type": sys.platform, - }) - response.raise_for_status() + response = self._session.get( + "{}/combadge".format(self._url), + timeout=_TIMEOUT, + params={"combadge_version": combadge_version, "os_type": sys.platform,}, + ) + raise_for_status(response) - if combadge_version == 'v1': # python version + if combadge_version == "v1": # python version self._combadge = CombadgePython.from_response(response) - elif combadge_version == 'v2': # rust version + elif combadge_version == "v2": # rust version self._combadge = CombadgeRust.from_response(response) else: raise Exception("Wrong combadge type") return self._combadge @property - def session(self): + def session(self) -> requests.Session: return self._session - def __del__(self): + def __del__(self) -> None: self._session.close() @property - def url(self): + def url(self) -> str: return self._url - def beam_up(self, directory, combadge_version=None, email=None, beam_type=None, tags=None, return_beam_object=False): + def beam_up( + self, + directory: str, + combadge_version: typing.Optional[str] = None, + email: typing.Optional[str] = None, + beam_type: typing.Optional[str] = None, + tags: typing.Optional[typing.List[str]] = None, + return_beam_object: bool = False, + ) -> typing.Union["Beam", int]: """Beam up the specified local directory to Scotty. :param str directory: Local directory to beam. @@ -153,47 +208,67 @@ def beam_up(self, directory, combadge_version=None, email=None, beam_type=None, combadge_version = self._get_combadge_version(version_override=combadge_version) directory = os.path.abspath(directory) response = self._session.get("{}/info".format(self._url), timeout=_TIMEOUT) - response.raise_for_status() - transporter_host = response.json()['transporter'] + raise_for_status(response) + transporter_host = response.json()["transporter"] beam = { - 'directory': directory, - 'host': socket.gethostname(), - 'auth_method': 'independent', - 'type': beam_type, - 'combadge_version': combadge_version, - 'os_type': sys.platform, - } + "directory": directory, + "host": socket.gethostname(), + "auth_method": "independent", + "type": beam_type, + "combadge_version": combadge_version, + "os_type": sys.platform, + } # type: JSON if email: - beam['email'] = email + beam["email"] = email if tags: - beam['tags'] = tags + beam["tags"] = tags - response = self._session.post("{}/beams".format(self._url), data=json.dumps({'beam': beam}), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.post( + "{}/beams".format(self._url), + data=json.dumps({"beam": beam}), + timeout=_TIMEOUT, + ) + raise_for_status(response) beam_data = response.json() - beam_id = beam_data['beam']['id'] + beam_id = beam_data["beam"]["id"] # type: int combadge = self._get_combadge(combadge_version) - combadge.run(beam_id=beam_id, directory=directory, transporter_host=transporter_host) + combadge.run( + beam_id=beam_id, directory=directory, transporter_host=transporter_host + ) if return_beam_object: - return Beam.from_json(self, beam_data['beam']) + return Beam.from_json(self, beam_data["beam"]) else: - return beam_data['beam']['id'] + return beam_id - def _get_combadge_version(self, version_override=None): + def _get_combadge_version( + self, version_override: typing.Optional[str] = None + ) -> str: return ( - version_override or - (self._combadge and self._combadge.version) or - _DEFAULT_COMBADGE_VERSION + version_override + or (self._combadge and self._combadge.version) + or _DEFAULT_COMBADGE_VERSION ) - def initiate_beam(self, user, host, directory, password=None, rsa_key=None, email=None, beam_type=None, - stored_key=None, tags=None, return_beam_object=False, combadge_version=None): + def initiate_beam( + self, + user: str, + host: str, + directory: str, + password: typing.Optional[str] = None, + rsa_key: typing.Optional[str] = None, + email: typing.Optional[str] = None, + beam_type: typing.Optional[str] = None, + stored_key: typing.Optional[str] = None, + tags: typing.Optional[typing.List[str]] = None, + return_beam_object: bool = False, + combadge_version: typing.Optional[str] = None, + ) -> typing.Union["Beam", int]: """Order scotty to beam the specified directory from the specified host. :param str user: The username in the remote machine. @@ -213,182 +288,229 @@ def initiate_beam(self, user, host, directory, password=None, rsa_key=None, emai :return: the beam id.""" combadge_version = self._get_combadge_version(version_override=combadge_version) if len([x for x in (password, rsa_key, stored_key) if x]) != 1: - raise Exception("Either password, rsa_key or stored_key should be specified") + raise Exception( + "Either password, rsa_key or stored_key should be specified" + ) if rsa_key: - auth_method = 'rsa' + auth_method = "rsa" elif password: - auth_method = 'password' + auth_method = "password" elif stored_key: - auth_method = 'stored_key' + auth_method = "stored_key" else: raise Exception() beam = { - 'directory': directory, - 'host': host, - 'user': user, - 'ssh_key': rsa_key, - 'stored_key': stored_key, - 'password': password, - 'type': beam_type, - 'auth_method': auth_method, - 'combadge_version': combadge_version, - } + "directory": directory, + "host": host, + "user": user, + "ssh_key": rsa_key, + "stored_key": stored_key, + "password": password, + "type": beam_type, + "auth_method": auth_method, + "combadge_version": combadge_version, + } # type: JSON if tags: - beam['tags'] = tags + beam["tags"] = tags if email: - beam['email'] = email + beam["email"] = email - response = self._session.post("{0}/beams".format(self._url), data=json.dumps({'beam': beam}), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.post( + "{0}/beams".format(self._url), + data=json.dumps({"beam": beam}), + timeout=_TIMEOUT, + ) + raise_for_status(response) beam_data = response.json() if return_beam_object: - return Beam.from_json(self, beam_data['beam']) + return Beam.from_json(self, beam_data["beam"]) else: - return beam_data['beam']['id'] + beam_id = beam_data["beam"]["id"] # type: int + return beam_id - def add_tag(self, beam_id, tag): + def add_tag(self, beam_id: int, tag: str) -> None: """Add the specified tag on the specified beam id. :param int beam_id: Beam ID. :param str tag: Tag name.""" - response = self._session.post("{0}/beams/{1}/tags/{2}".format(self._url, beam_id, tag), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.post( + "{0}/beams/{1}/tags/{2}".format(self._url, beam_id, tag), timeout=_TIMEOUT + ) + raise_for_status(response) - def remove_tag(self, beam_id, tag): + def remove_tag(self, beam_id: int, tag: str) -> None: """Remove the specified tag from the specified beam id. :param int beam_id: Beam ID. :param str tag: Tag name.""" - response = self._session.delete("{0}/beams/{1}/tags/{2}".format(self._url, beam_id, tag), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.delete( + "{0}/beams/{1}/tags/{2}".format(self._url, beam_id, tag), timeout=_TIMEOUT + ) + raise_for_status(response) - def get_beam(self, beam_id): + def get_beam(self, beam_id: typing.Union[str, int]) -> "Beam": """Retrieve details about the specified beam. - :param int beam_id: Beam ID. + :param int beam_id: Beam ID or tag :rtype: :class:`.Beam`""" - response = self._session.get("{0}/beams/{1}".format(self._url, beam_id), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.get( + "{0}/beams/{1}".format(self._url, beam_id), timeout=_TIMEOUT + ) + raise_for_status(response) json_response = response.json() - return Beam.from_json(self, json_response['beam']) + return Beam.from_json(self, json_response["beam"]) - def get_files(self, beam_id, filter_): + def get_files( + self, beam_id: int, filter_: typing.Optional[str] = None + ) -> typing.List[File]: response = self._session.get( "{0}/files".format(self._url), params={"beam_id": beam_id, "filter": filter_}, - timeout=_TIMEOUT) - response.raise_for_status() - return [File.from_json(self._session, f) for f in response.json()['files']] + timeout=_TIMEOUT, + ) + raise_for_status(response) + return [File.from_json(self._session, f) for f in response.json()["files"]] - def get_file(self, file_id): + def get_file(self, file_id: int) -> File: """Retrieve details about the specified file. :param int file_id: File ID. :rtype: :class:`.File`""" - response = self._session.get("{0}/files/{1}".format(self._url, file_id), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.get( + "{0}/files/{1}".format(self._url, file_id), timeout=_TIMEOUT + ) + raise_for_status(response) json_response = response.json() - return File.from_json(self._session, json_response['file']) + return File.from_json(self._session, json_response["file"]) - def get_beams_by_tag(self, tag): + def get_beams_by_tag(self, tag: str) -> typing.List[Beam]: """Retrieve the list of beams associated with the specified tag. :param str tag: The name of the tag. :return: a list of :class:`.Beam` objects. """ - response = self._session.get("{0}/beams?tag={1}".format(self._url, tag), timeout=_TIMEOUT) - response.raise_for_status() + response = self._session.get( + "{0}/beams?tag={1}".format(self._url, tag), timeout=_TIMEOUT + ) + raise_for_status(response) - ids = (b['id'] for b in response.json()['beams']) + ids = (b["id"] for b in response.json()["beams"]) return [self.get_beam(id_) for id_ in ids] - def sanity_check(self): + def sanity_check(self) -> None: """Check if this instance of Scotty is functioning. Raise an exception if something's wrong""" response = requests.get("{0}/info".format(self._url)) - response.raise_for_status() + raise_for_status(response) info = json.loads(response.text) - assert 'version' in info + assert "version" in info - def create_tracker(self, name, tracker_type, url, config): + def create_tracker( + self, name: str, tracker_type: str, url: str, config: JSON + ) -> int: data = { - 'tracker': { - 'name': name, - 'type': tracker_type, - 'url': url, - 'config': json.dumps(config) + "tracker": { + "name": name, + "type": tracker_type, + "url": url, + "config": json.dumps(config), } } - response = self._session.post("{}/trackers".format(self._url), data=json.dumps(data), timeout=_TIMEOUT) - response.raise_for_status() - return response.json()['tracker']['id'] + response = self._session.post( + "{}/trackers".format(self._url), data=json.dumps(data), timeout=_TIMEOUT + ) + raise_for_status(response) + tracker_id = response.json()["tracker"]["id"] # type: int + return tracker_id - def get_tracker_by_name(self, name): + def get_tracker_by_name(self, name: str) -> typing.Optional[JSON]: try: - response = self._session.get("{}/trackers/by_name/{}".format(self._url, name), timeout=_TIMEOUT) - response.raise_for_status() - return response.json()['tracker'] + response = self._session.get( + "{}/trackers/by_name/{}".format(self._url, name), timeout=_TIMEOUT + ) + raise_for_status(response) + tracker = response.json()["tracker"] # type: JSON + return tracker except requests.exceptions.HTTPError: return None - def get_tracker_id(self, name): - response = self._session.get("{}/trackers/by_name/{}".format(self._url, name), timeout=_TIMEOUT) - response.raise_for_status() - return response.json()['tracker']['id'] - - def create_issue(self, tracker_id, id_in_tracker): - data = { - 'issue': { - 'tracker_id': tracker_id, - 'id_in_tracker': id_in_tracker, - } - } - response = self._session.post("{}/issues".format(self._url), data=json.dumps(data), timeout=_TIMEOUT) - response.raise_for_status() - return response.json()['issue']['id'] + def get_tracker_id(self, name: str) -> int: + response = self._session.get( + "{}/trackers/by_name/{}".format(self._url, name), timeout=_TIMEOUT + ) + raise_for_status(response) + tracker_id = response.json()["tracker"]["id"] # type: int + return tracker_id + + def create_issue(self, tracker_id: int, id_in_tracker: str) -> int: + data = {"issue": {"tracker_id": tracker_id, "id_in_tracker": id_in_tracker,}} + response = self._session.post( + "{}/issues".format(self._url), data=json.dumps(data), timeout=_TIMEOUT + ) + raise_for_status(response) + issue_id = response.json()["issue"]["id"] # type: int + return issue_id - def delete_issue(self, issue_id): - response = self._session.delete("{}/issues/{}".format(self._url, issue_id), timeout=_TIMEOUT) - response.raise_for_status() + def delete_issue(self, issue_id: int) -> None: + response = self._session.delete( + "{}/issues/{}".format(self._url, issue_id), timeout=_TIMEOUT + ) + raise_for_status(response) - def get_issue_by_tracker(self, tracker_id, id_in_tracker): + def get_issue_by_tracker( + self, tracker_id: int, id_in_tracker: str + ) -> typing.Optional[JSON]: params = { - 'tracker_id': tracker_id, - 'id_in_tracker': id_in_tracker, + "tracker_id": tracker_id, + "id_in_tracker": id_in_tracker, } - response = self._session.get("{}/issues/get_by_tracker".format(self._url), params=params, timeout=_TIMEOUT) + response = self._session.get( + "{}/issues/get_by_tracker".format(self._url), + params=params, + timeout=_TIMEOUT, + ) try: - response.raise_for_status() - return response.json()['issue'] + raise_for_status(response) + issue = response.json()["issue"] # type: JSON + return issue except requests.exceptions.HTTPError: return None - def delete_tracker(self, tracker_id): - response = self._session.delete("{}/trackers/{}".format(self._url, tracker_id), timeout=_TIMEOUT) - response.raise_for_status() - - def update_tracker(self, tracker_id, name=None, url=None, config=None): + def delete_tracker(self, tracker_id: int) -> None: + response = self._session.delete( + "{}/trackers/{}".format(self._url, tracker_id), timeout=_TIMEOUT + ) + raise_for_status(response) + + def update_tracker( + self, + tracker_id: int, + name: typing.Optional[str] = None, + url: typing.Optional[str] = None, + config: typing.Optional[JSON] = None, + ) -> None: data = {} if name: - data['name'] = name + data["name"] = name if url: - data['url'] = url + data["url"] = url if config: - data['config'] = json.dumps(config) + data["config"] = json.dumps(config) response = self._session.put( "{}/trackers/{}".format(self._url, tracker_id), - data=json.dumps({'tracker': data}), - timeout=_TIMEOUT) - response.raise_for_status() + data=json.dumps({"tracker": data}), + timeout=_TIMEOUT, + ) + raise_for_status(response) diff --git a/scottypy/types.py b/scottypy/types.py new file mode 100644 index 0000000..e258a0e --- /dev/null +++ b/scottypy/types.py @@ -0,0 +1,3 @@ +import typing + +JSON = typing.Dict[str, typing.Any] diff --git a/scottypy/utils.py b/scottypy/utils.py new file mode 100644 index 0000000..60033ed --- /dev/null +++ b/scottypy/utils.py @@ -0,0 +1,29 @@ +import os + +import requests + + +def raise_for_status(response: requests.Response) -> None: + if 400 <= response.status_code < 500: + error_type = "Client" + elif 500 <= response.status_code < 600: + error_type = "Server" + else: + error_type = "" + if error_type: + try: + content = response.content.decode() + except UnicodeDecodeError: + content = "" + raise requests.HTTPError( + "{status_code}: {error_type} Error: {content}".format( + status_code=response.status_code, + error_type=error_type, + content=content, + ), + response=response, + ) + + +def fix_path_sep_for_current_platform(file_name: str) -> str: + return file_name.replace("\\", os.path.sep).replace("/", os.path.sep) diff --git a/setup.py b/setup.py index d0a5c27..b6c1f2e 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ author_email="roey.ghost@gmail.com", url="https://github.com/getslash/scottypy", version=__version__, # pylint: disable=E0602 - packages=find_packages(exclude=["tests"]), + packages=find_packages(exclude=["unittests"]), install_requires=install_requires, entry_points=dict( console_scripts=[ diff --git a/stubs/capacity/__init__.pyi b/stubs/capacity/__init__.pyi new file mode 100644 index 0000000..1720b71 --- /dev/null +++ b/stubs/capacity/__init__.pyi @@ -0,0 +1 @@ +from .capacity import Capacity as Capacity, EB as EB, EiB as EiB, GB as GB, GiB as GiB, KB as KB, KiB as KiB, MB as MB, MiB as MiB, PB as PB, PiB as PiB, TB as TB, TiB as TiB, bit as bit, byte as byte, from_string as from_string diff --git a/tests/__init__.py b/stubs/capacity/__version__.pyi similarity index 100% rename from tests/__init__.py rename to stubs/capacity/__version__.pyi diff --git a/stubs/capacity/capacity.pyi b/stubs/capacity/capacity.pyi new file mode 100644 index 0000000..40251bd --- /dev/null +++ b/stubs/capacity/capacity.pyi @@ -0,0 +1,56 @@ +from collections import namedtuple +from typing import Any + + +_StrCandidate = namedtuple('StrCandidate', ['weight', 'unit', 'str', 'unit_name']) + +class Capacity: + bits: Any = ... + def __init__(self, bits: Any) -> None: ... + def __bool__(self) -> bool: ... + def __hash__(self) -> Any: ... + def __eq__(self, other: Any) -> Any: ... + def __ne__(self, other: Any) -> Any: ... + def __lt__(self, other: Any) -> Any: ... + def __gt__(self, other: Any) -> Any: ... + def __le__(self, other: Any) -> Any: ... + def __ge__(self, other: Any) -> Any: ... + def __abs__(self) -> float: ... + def __neg__(self) -> float: ... + def __mul__(self, other: Any) -> "Capacity": ... + __rmul__: Any = ... + def __add__(self, other: Any) -> "Capacity": ... + __radd__: Any = ... + def __sub__(self, other: Any) -> "Capacity": ... + def __rsub__(self, other: Any) -> "Capacity": ... + def __div__(self, other: Any) -> "Capacity": ... + def __truediv__(self, other: Any) -> "Capacity": ... + def __floordiv__(self, other: Any) -> "Capacity": ... + def __rdiv__(self, other: Any) -> "Capacity": ... + __rtruediv__: Any = ... + __rfloordiv__: Any = ... + def __mod__(self, other: Any) -> "Capacity": ... + def __rmod__(self, other: Any) -> "Capacity": ... + def roundup(self, boundary: Any) -> "Capacity": ... + def rounddown(self, boundary: Any) -> "Capacity": ... + def __format__(self, capacity: Any) -> str: ... + + +def from_string(s: str) -> Capacity: + ... + + +EB: Capacity +EiB: Capacity +GB: Capacity +GiB: Capacity +KB: Capacity +KiB: Capacity +MB: Capacity +MiB: Capacity +PB: Capacity +PiB: Capacity +TB: Capacity +TiB: Capacity +bit: Capacity +byte: Capacity diff --git a/stubs/emport/__init__.pyi b/stubs/emport/__init__.pyi new file mode 100644 index 0000000..51cf5f5 --- /dev/null +++ b/stubs/emport/__init__.pyi @@ -0,0 +1,8 @@ +from typing import Any +from types import ModuleType + + +class NoInitFileFound(Exception): ... + +def import_file(filename: Any) -> ModuleType: ... +def set_package_name(directory: Any, name: Any) -> None: ... diff --git a/stubs/emport/__version__.pyi b/stubs/emport/__version__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/stubs/pact/__init__.pyi b/stubs/pact/__init__.pyi new file mode 100644 index 0000000..b1f5db8 --- /dev/null +++ b/stubs/pact/__init__.pyi @@ -0,0 +1,3 @@ +from .group import PactGroup as PactGroup +from .pact import Pact as Pact +from waiting.exceptions import TimeoutExpired as TimeoutExpired diff --git a/stubs/pact/_compat.pyi b/stubs/pact/_compat.pyi new file mode 100644 index 0000000..38f54d5 --- /dev/null +++ b/stubs/pact/_compat.pyi @@ -0,0 +1,12 @@ +from io import StringIO as StringIO +from typing import Any, Optional + +PY2: Any +zip: Any +xrange = range +iteritems: Any +itervalues: Any +integer_types: Any +string_types: Any + +def reraise(tp: Any, value: Any, tb: Optional[Any] = ...) -> None: ... diff --git a/stubs/pact/base.pyi b/stubs/pact/base.pyi new file mode 100644 index 0000000..0f0bbc1 --- /dev/null +++ b/stubs/pact/base.pyi @@ -0,0 +1,13 @@ +from ._compat import reraise as reraise +from typing import Any + +class PactBase: + def __init__(self, timeout_seconds: Any) -> None: ... + def get_timeout_exception(self, exc_info: Any) -> None: ... + def is_finished(self) -> bool: ... + def poll(self) -> bool: ... + def then(self, callback: Any, *args: Any, **kwargs: Any) -> "PactBase": ... + def lastly(self, callback: Any, *args: Any, **kwargs: Any) -> "PactBase": ... + def during(self, callback: Any, *args: Any, **kwargs: Any) -> "PactBase": ... + def on_timeout(self, callback: Any, *args: Any, **kwargs: Any) -> "PactBase": ... + def wait(self, **kwargs: Any) -> None: ... diff --git a/stubs/pact/group.pyi b/stubs/pact/group.pyi new file mode 100644 index 0000000..194dcc2 --- /dev/null +++ b/stubs/pact/group.pyi @@ -0,0 +1,9 @@ +from .base import PactBase as PactBase +from typing import Any, Optional + +class PactGroup(PactBase): + def __init__(self, pacts: Optional[Any] = ..., lazy: bool = ..., timeout_seconds: Optional[Any] = ...) -> None: ... + def __iadd__(self, other: Any) -> "PactGroup": ... + def __iter__(self) -> Any: ... + def get_timeout_exception(self, exc_info: Any) -> Any: ... + def add(self, pact: Any, absorb: bool = ...) -> None: ... diff --git a/stubs/pact/pact.pyi b/stubs/pact/pact.pyi new file mode 100644 index 0000000..d78fad6 --- /dev/null +++ b/stubs/pact/pact.pyi @@ -0,0 +1,11 @@ +from .base import PactBase as PactBase +from .group import PactGroup as PactGroup +from .utils import EdgeTriggered as EdgeTriggered +from typing import Any, Optional + +class Pact(PactBase): + msg: Any = ... + def __init__(self, msg: Any, timeout_seconds: Optional[Any] = ..., lazy: bool = ...) -> None: ... + def until(self, predicate: Any, *args: Any, **kwargs: Any) -> "Pact": ... + def group_with(self, other: Any) -> "PactGroup": ... + def __add__(self, other: Any) -> "Pact": ... diff --git a/stubs/pact/utils.pyi b/stubs/pact/utils.pyi new file mode 100644 index 0000000..16fc7e0 --- /dev/null +++ b/stubs/pact/utils.pyi @@ -0,0 +1,5 @@ +from typing import Any + +class EdgeTriggered: + def __init__(self, callback: Any, args: Any, kwargs: Any) -> None: ... + def satisfied(self) -> bool: ... diff --git a/stubs/waiting/__init__.pyi b/stubs/waiting/__init__.pyi new file mode 100644 index 0000000..e5333f3 --- /dev/null +++ b/stubs/waiting/__init__.pyi @@ -0,0 +1,21 @@ +from typing import Any, Optional + +def wait(*args: Any, **kwargs: Any) -> bool: ... +def iterwait(predicate: Any, timeout_seconds: Optional[Any] = ..., sleep_seconds: int = ..., result: Optional[Any] = ..., waiting_for: Optional[Any] = ..., on_poll: Optional[Any] = ..., expected_exceptions: Any = ...) -> None: ... + +class _SleepToggle: + enabled: bool = ... + def __call__(self) -> None: ... + +class _Result: + result: Any = ... + +class Aggregate: + predicates: Any = ... + def __init__(self, predicates: Any) -> None: ... + +class ANY(Aggregate): + def __call__(self) -> bool: ... + +class ALL(Aggregate): + def __call__(self) -> bool: ... diff --git a/stubs/waiting/__version__.pyi b/stubs/waiting/__version__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/stubs/waiting/deadlines.pyi b/stubs/waiting/deadlines.pyi new file mode 100644 index 0000000..0232c4c --- /dev/null +++ b/stubs/waiting/deadlines.pyi @@ -0,0 +1,15 @@ +from typing import Any + +class Deadline: + def is_expired(self) -> None: ... + def get_num_seconds_remaining(self) -> None: ... + +class Within(Deadline): + def __init__(self, seconds: Any) -> None: ... + def is_expired(self): ... + def get_num_seconds_remaining(self): ... + +class Whenever(Deadline): + def is_expired(self): ... + +def make_deadline(seconds: Any): ... diff --git a/stubs/waiting/exceptions.pyi b/stubs/waiting/exceptions.pyi new file mode 100644 index 0000000..c045c94 --- /dev/null +++ b/stubs/waiting/exceptions.pyi @@ -0,0 +1,11 @@ +from typing import Any + +class TimeoutExpired(Exception): + def __init__(self, timeout_seconds: Any, what: Any) -> None: ... + +class IllegalArgumentError(ValueError): ... + +class NestedStopIteration(Exception): + exc_info: Any = ... + def __init__(self, exc_info: Any) -> None: ... + def reraise(self) -> None: ... diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index c884609..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -slash -scotty diff --git a/tests/slashconf.py b/tests/slashconf.py deleted file mode 100644 index 517ef10..0000000 --- a/tests/slashconf.py +++ /dev/null @@ -1,16 +0,0 @@ -import tempfile -import shutil -from slash import fixture -from scottypy import Scotty - - -@fixture -def scotty(): - return Scotty("http://localhost:8000") - - -@fixture -def tempdir(this): - d = tempfile.mkdtemp() - this.add_cleanup(lambda: shutil.rmtree(d)) - return d diff --git a/tests/test_sanity.py b/tests/test_sanity.py deleted file mode 100644 index 2929d39..0000000 --- a/tests/test_sanity.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import gzip -from contextlib import closing -from slash import parametrize - - -_LOG = """I can see what you see not -Vision milky, then eyes rot. -When you turn, they will be gone, -Whispering their hidden song. -Then you see what cannot be -Shadows move where light should be. -Out of darkness, out of mind, -Cast down into the Halls of the Blind.""" - - -def _local_beam(scotty, email): - return scotty.beam_up('build', email=email) - - -def _remote_beam_password(scotty, email): - return scotty.initiate_beam('vagrant', '192.168.50.4', '/home/vagrant', password='vagrant', email=email) - - -def _remote_beam_rsa(scotty, email): - with open(os.path.expanduser("~/.vagrant.d/insecure_private_key"), "r") as f: - key = f.read() - - return scotty.initiate_beam('vagrant', '192.168.50.4', '/home/vagrant', rsa_key=key, email=email) - - -@parametrize('beam_function', [_local_beam, _remote_beam_password, _remote_beam_rsa]) -@parametrize('email', [None, 'roeyd@infinidat.com']) -def test_sanity(scotty, beam_function, email): - beam_id = beam_function(scotty, email) - scotty.add_tag(beam_id, 'tag/with/slashes') - assert beam_id in [b.id for b in scotty.get_beams_by_tag('tag/with/slashes')] - scotty.remove_tag(beam_id, 'tag/with/slashes') - assert beam_id not in [b.id for b in scotty.get_beams_by_tag('tag/with/slashes')] - beam = scotty.get_beam(beam_id) - pact = beam.get_pact() - pact.wait() - assert beam.completed - - -def test_single_file(scotty, tempdir): - file_name = "something.log" - path = os.path.join(tempdir, file_name) - with open(path, "w") as log_file: - log_file.write(_LOG) - - beam_id = scotty.beam_up(path) - beam = scotty.get_beam(beam_id) - files = list(beam.iter_files()) - assert len(files) == 1 - - beamed_file = files[0].storage_name - full_path = os.path.join("/var/scotty", beamed_file) - assert full_path.endswith(".log.gz"), full_path - assert file_name in beamed_file - with closing(gzip.GzipFile(full_path, "r")) as f: - content = f.read().decode("ASCII") # pylint: disable=no-member - - assert content == _LOG diff --git a/tests/test_scotty.py b/tests/test_scotty.py deleted file mode 100644 index 978e477..0000000 --- a/tests/test_scotty.py +++ /dev/null @@ -1,82 +0,0 @@ -# pylint: disable=redefined-outer-name -import time - -import pytest - -from scottypy.scotty import Scotty - -COMBADGE_VERSIONS = ["v1", "v2"] - - -@pytest.fixture() -def scotty_url(): - return "http://scotty-staging.lab.gdc.il.infinidat.com" - - -@pytest.fixture() -def directory(tmpdir): - with (tmpdir / "debug.log").open("w") as f: - f.write("") - return str(tmpdir) - - -@pytest.fixture() -def scotty(scotty_url): - return Scotty(url=scotty_url) - - -def test_scotty_url(scotty, scotty_url): - assert scotty._url == scotty_url - - -@pytest.mark.parametrize("combadge_version", COMBADGE_VERSIONS) -def test_prefetch_combadge(scotty, combadge_version): - scotty.prefetch_combadge(combadge_version=combadge_version) - - -@pytest.mark.parametrize("combadge_version", COMBADGE_VERSIONS) -def test_beam_up(scotty, combadge_version, directory): - email = "damram@infinidat.com" - beam_id = scotty.beam_up( - directory=directory, email=email, combadge_version=combadge_version - ) - beam = scotty.get_beam(beam_id) - assert beam.directory == directory - assert not beam.deleted - - -linux_host = "gdc-qa-io-005" -windows_host = "gdc-qa-io-349" - -remote_directories = [ - { - "host": linux_host, - "path": "/opt/infinidat/qa/logs/5704da68-588b-11ea-8e66-380025a4565f_0/001/vfs_logs/test/uproc/counters.nas--1", - "expected_num_files": 0, - }, - {"host": linux_host, "path": "/var/log/yum.log", "expected_num_files": 1}, - {"host": windows_host, "path": r"C:\Users\root\Documents\sandbox\debug.log", "expected_num_files": 1}, -] - - -@pytest.mark.parametrize("combadge_version", COMBADGE_VERSIONS) -@pytest.mark.parametrize("remote_directory", remote_directories) -def test_initiate_beam(scotty, combadge_version, remote_directory): - if remote_directory["host"] == windows_host and combadge_version == "v1": - raise pytest.skip("combadge v1 doesn't support windows") - email = "damram@infinidat.com" - beam_id = scotty.initiate_beam( - user="root", - host=remote_directory["host"], - directory=remote_directory["path"], - email=email, - combadge_version=combadge_version, - stored_key="1", - ) - beam = scotty.get_beam(beam_id) - while not beam.completed: - beam.update() - time.sleep(1) - assert not beam.error, beam.error - assert beam.directory == remote_directory["path"] - assert len(beam.get_files()) == remote_directory["expected_num_files"] diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a7f62a0..0000000 --- a/tox.ini +++ /dev/null @@ -1,7 +0,0 @@ -[tox] -envlist = py26,py27,py33,py34 - -[testenv] -commands= - pip install -r tests/requirements.txt - slash run tests diff --git a/unittests/fixtures/combadge b/unittests/fixtures/combadge index c58b671..c3182d4 100755 --- a/unittests/fixtures/combadge +++ b/unittests/fixtures/combadge @@ -20,4 +20,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/unittests/fixtures/combadge.py b/unittests/fixtures/combadge.py index 37547a5..7dca95f 100755 --- a/unittests/fixtures/combadge.py +++ b/unittests/fixtures/combadge.py @@ -3,8 +3,9 @@ def beam_up(beam_id, path, transporter_addr): - with open(os.path.join(path, 'output'), 'w') as f: - f.write("beam_id={beam_id}, path={path}, transporter_addr={transporter_addr}, version=v1".format( - beam_id=beam_id, path=path, transporter_addr=transporter_addr - )) - + with open(os.path.join(path, "output"), "w") as f: + f.write( + "beam_id={beam_id}, path={path}, transporter_addr={transporter_addr}, version=v1".format( + beam_id=beam_id, path=path, transporter_addr=transporter_addr + ) + ) diff --git a/unittests/test_scotty.py b/unittests/test_scotty.py index 35d5caf..1e8423d 100644 --- a/unittests/test_scotty.py +++ b/unittests/test_scotty.py @@ -10,7 +10,7 @@ import requests_mock from scottypy import Scotty -from scottypy.scotty import CombadgeRust, CombadgePython +from scottypy.scotty import CombadgePython, CombadgeRust class APICallLogger: @@ -32,7 +32,9 @@ def isolate(self): self.calls = old_calls def get_single_call_or_raise(self): - assert len(self.calls) == 1, "Expected one call, got {calls}".format(calls=self.calls) + assert len(self.calls) == 1, "Expected one call, got {calls}".format( + calls=self.calls + ) return self.calls[0] def _assert_urls_equal(self, actual_url, expected_url): @@ -41,15 +43,19 @@ def _assert_urls_equal(self, actual_url, expected_url): assert actual_url_parsed.scheme == expected_url_parsed.scheme assert actual_url_parsed.netloc == expected_url_parsed.netloc assert actual_url_parsed.path == expected_url_parsed.path - assert urllib.parse.parse_qs(actual_url_parsed.query) == urllib.parse.parse_qs(expected_url_parsed.query) + assert urllib.parse.parse_qs(actual_url_parsed.query) == urllib.parse.parse_qs( + expected_url_parsed.query + ) assert actual_url_parsed.fragment == expected_url_parsed.fragment def assert_urls_equal_to(self, expected_urls): - assert len(expected_urls) == len(self.calls), "Expected {} calls, got {} instead".format( + assert len(expected_urls) == len( + self.calls + ), "Expected {} calls, got {} instead".format( len(expected_urls), len(self.calls) ) for call, expected_url in zip(self.calls, expected_urls): - actual_url = call['url'] + actual_url = call["url"] self._assert_urls_equal(actual_url, expected_url) @@ -63,9 +69,11 @@ def combadge(api_call_logger, request, context): elif combadge_version == "v2" and os_type == "linux": file_name = "combadge" else: - raise ValueError("Unknown combadge version {combadge_version}".format( - combadge_version=combadge_version - )) + raise ValueError( + "Unknown combadge version {combadge_version}".format( + combadge_version=combadge_version + ) + ) with open(os.path.join(fixtures_folder, file_name), "rb") as f: return f.read() @@ -104,7 +112,9 @@ def api_call_logger(): def scotty(api_call_logger): url = "http://mock-scotty" with requests_mock.Mocker() as m: - m.get("{url}/combadge".format(url=url), content=partial(combadge, api_call_logger)) + m.get( + "{url}/combadge".format(url=url), content=partial(combadge, api_call_logger) + ) m.get("{url}/info".format(url=url), json={"transporter": "mock-transporter"}) m.post("{url}/beams".format(url=url), json=partial(beams, api_call_logger)) yield Scotty(url) @@ -152,15 +162,16 @@ def _validate_beam_up(*, directory, combadge_version): @pytest.mark.parametrize("combadge_version", ["v1", "v2"]) -def test_prefetch_and_then_beam_up_doesnt_download_combadge_again(scotty, directory, combadge_version, api_call_logger): +def test_prefetch_and_then_beam_up_doesnt_download_combadge_again( + scotty, directory, combadge_version, api_call_logger +): with api_call_logger.isolate(): scotty.prefetch_combadge(combadge_version=combadge_version) expected_combadge_url = ( "http://mock-scotty/" "combadge?" "combadge_version={combadge_version}" - "&os_type=linux" - .format(combadge_version=combadge_version) + "&os_type=linux".format(combadge_version=combadge_version) ) api_call_logger.assert_urls_equal_to([expected_combadge_url]) with api_call_logger.isolate(): @@ -171,47 +182,61 @@ def test_prefetch_and_then_beam_up_doesnt_download_combadge_again(scotty, direct _validate_beam_up(combadge_version=combadge_version, directory=directory) -def test_prefetch_and_then_beam_different_version_downloads_combadge_again(scotty, directory, api_call_logger): +def test_prefetch_and_then_beam_different_version_downloads_combadge_again( + scotty, directory, api_call_logger +): with api_call_logger.isolate(): - scotty.prefetch_combadge(combadge_version='v1') - expected_combadge_url = "http://mock-scotty/combadge?combadge_version=v1&os_type=linux" + scotty.prefetch_combadge(combadge_version="v1") + expected_combadge_url = ( + "http://mock-scotty/combadge?combadge_version=v1&os_type=linux" + ) api_call_logger.assert_urls_equal_to([expected_combadge_url]) with api_call_logger.isolate(): scotty.beam_up( - directory=directory, combadge_version='v2', + directory=directory, combadge_version="v2", ) - api_call_logger.assert_urls_equal_to([ - "http://mock-scotty/beams", - "http://mock-scotty/combadge?combadge_version=v2&os_type=linux", - ]) - _validate_beam_up(combadge_version='v2', directory=directory) + api_call_logger.assert_urls_equal_to( + [ + "http://mock-scotty/beams", + "http://mock-scotty/combadge?combadge_version=v2&os_type=linux", + ] + ) + _validate_beam_up(combadge_version="v2", directory=directory) -def test_prefetch_and_then_beam_different_version_twice_downloads_combadge_again_once(scotty, directory, api_call_logger): +def test_prefetch_and_then_beam_different_version_twice_downloads_combadge_again_once( + scotty, directory, api_call_logger +): with api_call_logger.isolate(): - scotty.prefetch_combadge(combadge_version='v1') - expected_combadge_url = "http://mock-scotty/combadge?combadge_version=v1&os_type=linux" + scotty.prefetch_combadge(combadge_version="v1") + expected_combadge_url = ( + "http://mock-scotty/combadge?combadge_version=v1&os_type=linux" + ) api_call_logger.assert_urls_equal_to([expected_combadge_url]) with api_call_logger.isolate(): scotty.beam_up( - directory=directory, combadge_version='v2', + directory=directory, combadge_version="v2", + ) + api_call_logger.assert_urls_equal_to( + [ + "http://mock-scotty/beams", + "http://mock-scotty/combadge?combadge_version=v2&os_type=linux", + ] ) - api_call_logger.assert_urls_equal_to([ - "http://mock-scotty/beams", - "http://mock-scotty/combadge?combadge_version=v2&os_type=linux", - ]) - _validate_beam_up(combadge_version='v2', directory=directory) + _validate_beam_up(combadge_version="v2", directory=directory) with api_call_logger.isolate(): scotty.beam_up( - directory=directory, combadge_version='v2', + directory=directory, combadge_version="v2", ) api_call_logger.assert_urls_equal_to(["http://mock-scotty/beams"]) - _validate_beam_up(combadge_version='v2', directory=directory) + _validate_beam_up(combadge_version="v2", directory=directory) @pytest.mark.parametrize("combadge_version", ["v1", "v2"]) -def test_prefetch_and_then_beam_up_without_explicit_version_uses_prefetched(scotty, directory, api_call_logger, combadge_version): +def test_prefetch_and_then_beam_up_without_explicit_version_uses_prefetched( + scotty, directory, api_call_logger, combadge_version +): with api_call_logger.isolate(): scotty.prefetch_combadge(combadge_version=combadge_version) expected_combadge_url = "http://mock-scotty/combadge?combadge_version={combadge_version}&os_type=linux".format( @@ -219,15 +244,15 @@ def test_prefetch_and_then_beam_up_without_explicit_version_uses_prefetched(scot ) api_call_logger.assert_urls_equal_to([expected_combadge_url]) with api_call_logger.isolate(): - scotty.beam_up( - directory=directory - ) + scotty.beam_up(directory=directory) api_call_logger.assert_urls_equal_to(["http://mock-scotty/beams"]) _validate_beam_up(combadge_version=combadge_version, directory=directory) @pytest.mark.parametrize("combadge_version", ["v1", "v2"]) -def test_prefetch_and_then_initiate_beam_without_explicit_version_uses_prefetched(scotty, directory, api_call_logger, combadge_version): +def test_prefetch_and_then_initiate_beam_without_explicit_version_uses_prefetched( + scotty, directory, api_call_logger, combadge_version +): with api_call_logger.isolate(): scotty.prefetch_combadge(combadge_version=combadge_version) expected_combadge_url = "http://mock-scotty/combadge?combadge_version={combadge_version}&os_type=linux".format( @@ -236,12 +261,14 @@ def test_prefetch_and_then_initiate_beam_without_explicit_version_uses_prefetche api_call_logger.assert_urls_equal_to([expected_combadge_url]) with api_call_logger.isolate(): scotty.initiate_beam( - user="mock-user", - host="mock-host", - stored_key="1", - directory=directory + user="mock-user", host="mock-host", stored_key="1", directory=directory + ) + assert ( + api_call_logger.get_single_call_or_raise()["json"]["beam"][ + "combadge_version" + ] + == combadge_version ) - assert api_call_logger.get_single_call_or_raise()["json"]["beam"]["combadge_version"] == combadge_version @pytest.mark.parametrize("combadge_version", ["v1", "v2"]) diff --git a/unittests/test_utils.py b/unittests/test_utils.py new file mode 100644 index 0000000..0ad8d1c --- /dev/null +++ b/unittests/test_utils.py @@ -0,0 +1,45 @@ +import os +from http import HTTPStatus + +import pytest +from requests import HTTPError + +from scottypy import utils + + +class MockResponse: + def __init__(self, *, status_code, content=b""): + self.content = content + self.status_code = status_code + + +def test_raise_for_status_client_error(): + with pytest.raises(HTTPError, match="409: Client Error: duplicate key"): + utils.raise_for_status( + MockResponse(status_code=HTTPStatus.CONFLICT, content=b"duplicate key") + ) + + +def test_raise_for_status_server_error(): + with pytest.raises(HTTPError, match="500: Server Error: should not be x"): + utils.raise_for_status( + MockResponse( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, content=b"should not be x" + ) + ) + + +def test_raise_for_status_no_error(): + utils.raise_for_status(MockResponse(status_code=HTTPStatus.OK)) + + +def test_fix_path_sep_for_current_platform_windows_path(): + assert utils.fix_path_sep_for_current_platform(r"a\b\c") == os.path.join( + "a", "b", "c" + ) + + +def test_fix_path_sep_for_current_platform_linux_path(): + assert utils.fix_path_sep_for_current_platform("a/b/c") == os.path.join( + "a", "b", "c" + )