From 39fd2d41a98769ea9d31c1e1882182af73654870 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 8 Jul 2013 14:58:48 -0700 Subject: [PATCH 01/48] tronstore first steps and some unit tests, incomplete --- tests/serialize/runstate/statemanager_test.py | 6 +- tron/config/config_parse.py | 7 +- tron/config/manager.py | 2 +- tron/config/schema.py | 5 +- tron/serialize/runstate/__init__.py | 3 +- tron/serialize/runstate/tronstore/__init__.py | 1 + tron/serialize/runstate/tronstore/chunking.py | 19 ++ tron/serialize/runstate/tronstore/messages.py | 102 ++++++++ .../serialize/runstate/tronstore/msg_enums.py | 4 + .../runstate/tronstore/parallelstore.py | 95 +++++++ tron/serialize/runstate/tronstore/process.py | 98 ++++++++ tron/serialize/runstate/tronstore/store.py | 232 ++++++++++++++++++ .../serialize/runstate/tronstore/transport.py | 71 ++++++ tron/serialize/runstate/tronstore/tronstore | 90 +++++++ 14 files changed, 729 insertions(+), 6 deletions(-) create mode 100644 tron/serialize/runstate/tronstore/__init__.py create mode 100644 tron/serialize/runstate/tronstore/chunking.py create mode 100644 tron/serialize/runstate/tronstore/messages.py create mode 100644 tron/serialize/runstate/tronstore/msg_enums.py create mode 100644 tron/serialize/runstate/tronstore/parallelstore.py create mode 100644 tron/serialize/runstate/tronstore/process.py create mode 100644 tron/serialize/runstate/tronstore/store.py create mode 100644 tron/serialize/runstate/tronstore/transport.py create mode 100755 tron/serialize/runstate/tronstore/tronstore diff --git a/tests/serialize/runstate/statemanager_test.py b/tests/serialize/runstate/statemanager_test.py index e793be471..6d7851fd2 100644 --- a/tests/serialize/runstate/statemanager_test.py +++ b/tests/serialize/runstate/statemanager_test.py @@ -21,7 +21,9 @@ def test_from_config_shelve(self): thefilename = 'thefilename' config = schema.ConfigState( store_type='shelve', name=thefilename, buffer_size=0, - connection_details=None) + transport_method='pickle', + connection_details=None, + db_store_method=None) manager = PersistenceManagerFactory.from_config(config) store = manager._impl assert_equal(store.filename, config.name) @@ -202,4 +204,4 @@ def test_restore(self): if __name__ == "__main__": - run() \ No newline at end of file + run() diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 897b4c673..3f08b7752 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -392,14 +392,19 @@ class ValidateStatePersistence(Validator): defaults = { 'buffer_size': 1, 'connection_details': None, + 'db_store_method': 'msgpack', } validators = { 'name': valid_string, 'store_type': config_utils.build_enum_validator( schema.StatePersistenceTypes), + 'transport_method': config_utils.build_enum_validator( + schema.StateTransportTypes), 'connection_details': valid_string, 'buffer_size': valid_int, + 'db_store_method': config_utils.build_enum_validator( + schema.StateTransportTypes), } def post_validation(self, config, config_context): @@ -426,7 +431,7 @@ def validate_jobs_and_services(config, config_context): config_utils.unique_names(fmt_string, config['jobs'], config['services']) -DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', None, 1) +DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', 'pickle', None, 1, None) DEFAULT_NODE = ValidateNode().do_shortcut('localhost') diff --git a/tron/config/manager.py b/tron/config/manager.py index 68e4dcb4c..6b28f8712 100644 --- a/tron/config/manager.py +++ b/tron/config/manager.py @@ -134,4 +134,4 @@ def create_new_config(path, master_content): manager = ConfigManager(path) manager.manifest.create() filename = manager.get_filename_from_manifest(schema.MASTER_NAMESPACE) - write_raw(filename , master_content) \ No newline at end of file + write_raw(filename , master_content) diff --git a/tron/config/schema.py b/tron/config/schema.py index ebb385bf9..f65b76726 100644 --- a/tron/config/schema.py +++ b/tron/config/schema.py @@ -90,9 +90,11 @@ def config_object_factory(name, required=None, optional=None): [ 'name', 'store_type', + 'transport_method', ],[ 'connection_details', - 'buffer_size' + 'buffer_size', + 'db_store_method', ]) @@ -151,6 +153,7 @@ def config_object_factory(name, required=None, optional=None): StatePersistenceTypes = Enum.create('shelve', 'sql', 'mongo', 'yaml') +StateTransportTypes = Enum.create('json', 'pickle', 'msgpack', 'yaml') ActionRunnerTypes = Enum.create('none', 'subprocess') diff --git a/tron/serialize/runstate/__init__.py b/tron/serialize/runstate/__init__.py index a04999886..39127b2a8 100644 --- a/tron/serialize/runstate/__init__.py +++ b/tron/serialize/runstate/__init__.py @@ -2,5 +2,6 @@ # State types JOB_STATE = 'job_state' +JOB_RUN_STATE = 'job_run_state' SERVICE_STATE = 'service_state' -MCP_STATE = 'mcp_state' \ No newline at end of file +MCP_STATE = 'mcp_state' diff --git a/tron/serialize/runstate/tronstore/__init__.py b/tron/serialize/runstate/tronstore/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tron/serialize/runstate/tronstore/__init__.py @@ -0,0 +1 @@ + diff --git a/tron/serialize/runstate/tronstore/chunking.py b/tron/serialize/runstate/tronstore/chunking.py new file mode 100644 index 000000000..fc74418d0 --- /dev/null +++ b/tron/serialize/runstate/tronstore/chunking.py @@ -0,0 +1,19 @@ +CHUNK_SIGNING_STR = '\xDE\xAD\xBE\xEF\x00' + +class StoreChunkHandler(object): + + def __init__(self): + self.chunk = '' + + def sign(self, data): + return (data + CHUNK_SIGNING_STR) + + def handle(self, data): + self.chunk += data + chunks = self.chunk.split(CHUNK_SIGNING_STR) + # if the chunk doesn't end with the signed string, the message was + # incomplete and should be saved for the next time .handle is called. + if not self.chunk.endswith(CHUNK_SIGNING_STR): + self.chunk = chunks[-1:] + chunks = chunks[:-1] + return chunks diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py new file mode 100644 index 000000000..4a5821ce7 --- /dev/null +++ b/tron/serialize/runstate/tronstore/messages.py @@ -0,0 +1,102 @@ +from tron.serialize.runstate.tronstore.transport import JSONTransport, cPickleTransport, MsgPackTransport, YamlTransport + +transport_class_map = { + 'json': JSONTransport, + 'pickle': cPickleTransport, + 'msgpack': MsgPackTransport, + 'yaml': YamlTransport +} + +MAX_MSG_ID = 2**32 - 1 + + +class StoreRequestFactory(object): + + def __init__(self, method): + self.serializer = transport_class_map[method] + self.id_counter = 1 + + def _increment_counter(self): + return self.id+1 if not self.id == MAX_MSG_ID else 0 + + def build(self, req_type, data_type, data): + new_request = StoreRequest(self.id_counter, req_type, data, self.serializer) + self.id_counter = self._increment_counter() + return new_request + + def rebuild(self, msg): + return StoreRequest.from_message(self.serializer.deserialize(msg), self.serializer) + + def update_method(self, new_method): + """Update the method used for message serialization.""" + self.serializer = transport_class_map[new_method] + + def get_method(self): + return self.serializer + + +class StoreResponseFactory(object): + + def __init__(self, method): + self.serializer = transport_class_map[method] + + def build(self, success, req_id, data): + new_request = StoreResponse(req_id, success, data, self.serializer) + return new_request + + def rebuild(self, msg): + return StoreResponse.from_message(self.serializer.deserialize(msg), self.serializer) + + def update_method(self, new_method): + """Update the method used for message serialization.""" + self.serializer = transport_class_map[new_method] + + def get_method(self): + return self.serializer + + +class StoreRequest(object): + + def __init__(self, req_id, req_type, data_type, data, method): + self.id = req_id + self.req_type = req_type + self.data = data + self.data_type = data_type + self.method = method + self.serialized = self.get_serialized() + + @classmethod + def from_message(cls, msg_data, method): + req_id, req_type, data_type, data = msg_data + return cls(req_id, req_type, data_type, data, method) + + def update_method(self, new_method): + """Update the method used for message serialization.""" + self.method = transport_class_map['new_method'] + self.serialized = self.get_serialized() + + def get_serialized(self): + return self.method.serialize((self.id, self.req_type, self.data_type, self.data)) + + +class StoreResponse(object): + + def __init__(self, req_id, success, data, method): + self.id = req_id + self.success = success + self.data = data + self.method = method + self.serialized = self.get_serialized() + + @classmethod + def from_message(cls, msg_data, method): + req_id, success, data = msg_data + return cls(req_id, success, data, method) + + def update_method(self, new_method): + """Update the method used for message serialization.""" + self.method = transport_class_map['new_method'] + self.serialized = self.get_serialized() + + def get_serialized(self): + return self.method.serialize((self.id, self.success, self.data)) diff --git a/tron/serialize/runstate/tronstore/msg_enums.py b/tron/serialize/runstate/tronstore/msg_enums.py new file mode 100644 index 000000000..30c2f29ac --- /dev/null +++ b/tron/serialize/runstate/tronstore/msg_enums.py @@ -0,0 +1,4 @@ +REQUEST_SAVE = 10 +REQUEST_RESTORE = 11 +# REQUEST_CONFIG = 12 # Probably not needed... +# REQUEST_SHUTDOWN = 13 # Not needed, using SIGINT diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py new file mode 100644 index 000000000..b5d073c12 --- /dev/null +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -0,0 +1,95 @@ +import itertools +import operator +import logging + +from twisted.internet import reactor +from tron.serialize.runstate.tronstore.process import StoreProcessProtocol +from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory +from tron.serialize.runstate.tronstore import msg_enums + +log = logging.getLogger(__name__) + + +class ParallelKey(object): + __slots__ = ['type', 'iden'] + + def __init__(self, type, iden): + self.type = type + self.iden = iden + + @property + def key(self): + return str(self.iden) + + def __str__(self): + return "%s %s" % (self.type, self.iden) + + def __eq__(self, other): + return self.type == other.type and self.iden == other.iden + + def __hash__(self): + return hash(self.key) + +class ParallelStore(object): + """Persist state using a paralleled storing mechanism. This uses + the Twisted library to run the tronstore executable in a separate + process, and handles all communication between trond and tronstore. + + This class handles construction of all messages that need to be sent + to tronstore.""" + + def __init__(self, config): + self.config = config + self.request_factory = StoreRequestFactory(config.transport_method) + self.response_factory = StoreResponseFactory(config.transport_method) + self.process = StoreProcessProtocol(self.response_factory) + self.start_process() + + def start_process(self): + reactor.spawnProcess(self.process, "serialize/runstate/tronstore/tronstore", + ["tronstore", + self.config.name, + self.config.transport_method, + self.config.store_type, + self.config.connection_details, + self.config.db_store_method] + ) + reactor.run() + + def build_key(self, type, iden): + return ParallelKey(type, iden) + + def save(self, key_value_pairs): + for key, state_data in key_value_pairs: + request = self.request_factory.build(msg_enums.REQUEST_SAVE, key.type, (key.key, state_data)) + self.process.send_request(request) + + def restore_single(self, key): + request = self.request_factory.build(msg_enums.REQUEST_RESTORE, key.type, key.key) + response = self.process.send_request_get_response(request) + return response.data if response.successful else None + + def restore(self, keys): + items = itertools.izip(keys, (self.restore_single(key) for key in keys)) + return dict(itertools.ifilter(operator.itemgetter(1), items)) + + def cleanup(self): + self.process.shutdown() + shutdown = cleanup + + # This method may not be needed. From looking at the StateChangeWatcher + # implementation, it looks like it makes a completely new instance of a + # PersistentStateManager whenever the config is updated, which removes + # the need for changing config related things here (since a new instance + # of this class will be created anyway). + def config(self, new_config): + """Reconfigure the storing mechanism to use a new configuration.""" + self.config = new_config + self.request_factory.update_method(new_config.transport_method) + self.process.shutdown() + self.response_factory.update_method(new_config.transport_method) + self.process = StoreProcessProtocol(self.response_factory) + self.start_process() + + def __repr__(self): + return "ParallelStore" diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py new file mode 100644 index 000000000..e0e577dee --- /dev/null +++ b/tron/serialize/runstate/tronstore/process.py @@ -0,0 +1,98 @@ +import time +import logging +from threading import Semaphore + +from twisted.internet.protocol import ProcessProtocol +from twisted.internet import reactor +from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler + +log = logging.getLogger(__name__) + +class TronStoreError(Exception): + """Raised whenever tronstore exits for an unknown reason.""" + def __init__(self, code): + self.code = code + def __str__(self): + return repr(self.code) + +class StoreProcessProtocol(ProcessProtocol): + """The class that actually communicates with tronstore. This is a subclass + of the twisted ProcessProtocol class, which can run and asynchronously + communicate with a child proccess. The requests and responses are matched + together by the unique integer ids assigned to each request (which are also + present in responses). + """ + + SHUTDOWN_TIMEOUT = 5.0 + SHUTDOWN_SLEEP = 0.5 + + def __init__(self, response_factory): + self.response_factory = response_factory + self.chunker = StoreChunkHandler() + self.requests = {} + self.responses = {} + self.semaphores = {} # semaphores used for synchronization + self.is_shutdown = False + + def outRecieved(self, data): + responses = self.chunker.handle(data) + for response_str in responses: + response = self.response_factory.rebuild(response_str) + if response.id in self.requests: + if not response.success: + log.warn("tronstore request #%d failed. Request type was %d." % (response.id, self.requests[response.id].req_type)) + if response.id in self.monitors: + self.responses[response.id] = response + self.semaphores[response.id].release() + del self.requests[response.id] + + def processExited(self, reason): + if not self.is_shutdown: + raise TronStoreError(reason.getErrorMessage()) + + def processEnded(self, reason): + if not self.is_shutdown: + self.transport.loseConnection() + reactor.stop() + + def send_request(self, request): + """Send a request to tronstore and immediately return without + waiting for tronstore's response. + """ + if self.is_shutdown: + return + self.requests[request.id] = request + self.transport.write(self.chunker.sign(request.serialized)) + + def send_request_get_response(self, request): + """Send a request to tronstore, and block until tronstore responds + with the appropriate data. If the request was successful, we return + whatever data tronstore sent us, otherwise, None is returned. + """ + if self.is_shutdown: + return None + self.requests[request.id] = request + self.semaphores[request.id] = Semaphore(0) + self.transport.write(self.chunker.sign(request.serialized)) + self.semaphores[request.id].acquire() + del self.semaphores[request.id] + response = self.responses[request.id] + del self.responses[requests.id] + return response.data if response.success else None + + def shutdown(self): + """Shut down the process protocol. Waits for SHUTDOWN_TIMEOUT seconds + for all pending requests to get responses from tronstore, after which + it cuts the connection. It checks if all requests have been completed + every SHUTDOWN_SLEEP seconds. + + Calling this prevents ANY further requests being made to tronstore. + """ + self.is_shutdown = True + time_waited = 0 + while (not len(self.requests.items()) == 0) and time_waited < self.SHUTDOWN_TIMEOUT: + time.sleep(SHUTDOWN_SLEEP) # wait for all pending requests to finish + time_waited += SHUTDOWN_SLEEP + self.transport.signalProcess("INT") + self.transport.loseConnection() + self.reactor.stop() diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py new file mode 100644 index 000000000..0ea2c7540 --- /dev/null +++ b/tron/serialize/runstate/tronstore/store.py @@ -0,0 +1,232 @@ +import shelve +import urlparse +import os +from contextlib import contextmanager + +from tron.serialize.runstate.tronstore.messages import transport_class_map +from tron.serialize import runstate +from tron.config.config_utils import MAX_IDENTIFIER_LENGTH + + +class ShelveStore(object): + """Store state using python's built-in shelve module.""" + + def __init__(self, name, connection_details=None, serializer=None): + self.fname = name + self.shelve = shelve.open(self.fname) + + def save(self, key, state_data, data_type): + self.shelve['(%s__%s)' % (data_type, key)] = state_data + self.shelve.sync() + return True + + def restore(self, key, data_type): + value = self.shelve.get('(%s__%s)' % (data_type, key)) + return (True, value) if value else (False, None) + + def cleanup(self): + self.shelve.close() + + def __repr__(self): + return "ShelveStateStore('%s')" % self.filename + + +class SQLStore(object): + """Store state using SQLAlchemy.""" + + def __init__(self, name, connection_details, serializer): + import sqlalchemy as sql + global sql + assert sql + + self.name = name + self._connection = None + self.serializer = serializer + self.engine = sql.create_engine(connection_details) + self._setup_tables() + + def _setup_tables(self): + self._metadata = sql.MetaData() + self.job_state_table = sql.Table('job_state_data', self._metadata, + sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), + sql.Column('state_data', sql.Text)) + self.service_table = sql.Table('service_data', self._metadata, + sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), + sql.Column('state_data', sql.Text)) + self.job_run_table = sql.Table('job_run_data', self._metadata, + sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), + sql.Column('state_data', sql.Text)) + self.metadata_table = sql.Table('metadata_table', self._metadata, + sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), + sql.Column('state_data', sql.Text)) + + self._metadata.create_all(self.engine) + + @contextmanager + def connect(self): + if not self._connection or self._connection.closed: + self._connection = self.engine.connect() + yield self._connection + + def _get_table(self, data_type): + if data_type == runstate.JOB_STATE: + return self.job_state_table + elif data_type == runstate.JOB_RUN_STATE: + return self.job_run_table + elif data_type == runstate.SERVICE_STATE: + return self.service_table + elif data_type == runstate.MCP_STATE: + return self.metadata_table + else: + return None + + def save(self, key, state_data, data_type): + with self.connect() as conn: + table = self._get_table(data_type) + if table is None: + return False + state_data = self.serializer.serialize(state_data) + update_result = conn.execute( + table.update() + .where(table.c.key == key) + .values(state_data=state_data)) + if not update_result.rowcount: + conn.execute( + table.insert() + .values(key=key, state_data=state_data)) + return True + + def restore(self, key, data_type): + with self.connect() as conn: + table = self._get_table(data_type) + if table is None: + return (False, None) + result = conn.execute(sql.sql.select( + [table.c.state_data], + table.c.key == key) + ).fetchone() + return (True, self.serializer.deserialize(result[0])) if result else (False, None) + + def cleanup(self): + if self._connection: + self._connection.close() + + def __repr__(self): + return "SQLStore(%s)" % self.name + + +class MongoStore(object): + """Store state using mongoDB.""" + + JOB_COLLECTION = 'job_state_collection' + JOB_RUN_COLLECTION = 'job_run_state_collection' + SERVICE_COLLECTION = 'service_state_collection' + METADATA_COLLECTION = 'metadata_collection' + + TYPE_TO_COLLECTION_MAP = { + runstate.JOB_STATE: JOB_COLLECTION, + runstate.JOB_RUN_STATE: JOB_RUN_COLLECTION, + runstate.SERVICE_STATE: SERVICE_COLLECTION, + runstate.MCP_STATE: METADATA_COLLECTION + } + + def __init__(self, name, connection_details, serializer=None): + import pymongo + global pymongo + assert pymongo + + self.db_name = name + connection_params = self._parse_connection_details(connection_details) + self._connect(connection_params) + + def _connect(self, params): + hostname = params.get('hostname') + port = int(params.get('port')) + username = params.get('username') + password = params.get('password') + self.connection = pymongo.Connection(hostname, port) + self.db = self.connection[self.db_name] + if username and password: + self.db.authenticate(username, password) + + def _parse_connection_details(self, connection_details): + return dict(urlparse.parse_qsl(connection_details)) if connection_details else {} + + def save(self, key, state_data, data_type): + collection = self.db[self.TYPE_TO_COLLECTION_MAP[data_type]] + state_data['_id'] = key + collection.save(state_data) + return True + + def restore(self, key, data_type): + value = self.db[self.TYPE_TO_COLLECTION_MAP[data_type]].find_one(key) + return (True, value) if value else (False, None) + + def cleanup(self): + self.connection.disconnect() + + def __repr__(self): + return "MongoStore(%s)" % self.db_name + + +class YamlStore(object): + """Store state in a local YAML file. + + WARNING: Using this is NOT recommended, even moreso than the previous + version of this (yamlstore.py), since key/value pairs are now saved + INDIVIDUALLY rather than in batches, meaning saves are SLOOOOOOOW. + + Seriously, you probably shouldn't use this unless you're doing something + really trivial and/or want a readable Yaml file. + """ + + TYPE_MAPPING = { + runstate.JOB_STATE: 'jobs', + runstate.JOB_RUN_STATE: 'job_runs', + runstate.SERVICE_STATE: 'services', + runstate.MCP_STATE: runstate.MCP_STATE + } + + def __init__(self, filename, connection_details=None, serializer=None): + import yaml + global yaml + assert yaml + + self.filename = filename + if not os.path.exists(self.filename): + self.buffer = {} + else: + with open(self.filename, 'r') as fh: + self.buffer = yaml.load(fh) + + def save(self, key, state_data, data_type): + self.buffer.setdefault(self.TYPE_MAPPING[data_type], {})[key] = state_data + self._write_buffer() + return True + + def _write_buffer(self): + with open(self.filename, 'w') as fh: + yaml.dump(self.buffer, fh) + + def restore(self, key, data_type): + value = self.buffer.get(self.TYPE_MAPPING[data_type], {}).get(key) + return (True, value) if value else (False, None) + + def cleanup(self): + pass + + def __repr__(self): + return "YamlStore('%s')" % self.filename + + +store_class_map = { + "sql": SQLStore, + "shelve": ShelveStore, + "mongo": MongoStore, + "yaml": YamlStore +} + + +def build_store(name, store_type, connection_details, db_store_method): + trans_class = transport_class_map[db_store_method] if db_store_method != "None" else None + return store_class_map[store_type](name, connection_details, trans_class) diff --git a/tron/serialize/runstate/tronstore/transport.py b/tron/serialize/runstate/tronstore/transport.py new file mode 100644 index 000000000..d8fb80007 --- /dev/null +++ b/tron/serialize/runstate/tronstore/transport.py @@ -0,0 +1,71 @@ +import simplejson as json +import cPickle as pickle + +try: + import msgpack + no_msgpack = False +except ImportError: + no_msgpack = True + +try: + import yaml + no_yaml = False +except ImportError: + no_yaml = True + + +class TransportModuleError(Exception): + """Raised if a transport module is used without it being installed.""" + def __init__(self, code): + self.code = code + + def __str__(self): + return repr(self.code) + + +class JSONTransport(object): + @classmethod + def serialize(cls, data): + return json.dumps(data) + + @classmethod + def deserialize(cls, data_str): + return json.loads(data_str) + + +class cPickleTransport(object): + @classmethod + def serialize(cls, data): + return pickle.dumps(data) + + @classmethod + def deserialize(cls, data_str): + return pickle.loads(data_str) + + +class MsgPackTransport(object): + @classmethod + def serialize(cls, data): + if no_msgpack: + raise TransportModuleError('MessagePack not installed.') + return msgpack.packb(data, use_list=False) + + @classmethod + def deserialize(cls, data_str): + if no_msgpack: + raise TransportModuleError('MessagePack not installed.') + return msgpack.unpackb(data_str, use_list=False) + + +class YamlTransport(object): + @classmethod + def serialize(cls, data): + if no_yaml: + raise TransportModuleError('PyYaml not installed.') + return yaml.dump(data) + + @classmethod + def deserialize(cls, data_str): + if no_yaml: + raise TransportModuleError('PyYaml not installed.') + return yaml.load(data_str) diff --git a/tron/serialize/runstate/tronstore/tronstore b/tron/serialize/runstate/tronstore/tronstore new file mode 100755 index 000000000..6fe8af914 --- /dev/null +++ b/tron/serialize/runstate/tronstore/tronstore @@ -0,0 +1,90 @@ +#!/usr/bin/env python +import sys +import os +import signal +import time +from threading import Thread +from Queue import Queue, Empty + +from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory +import tron.serialize.runstate.tronstore.store +import tron.serialize.runstate.tronstore.msg_enums +from tron.serialize.tronstore.chunking import StoreChunkHandler + +is_shutdown = False + +def shutdown_handler(): + is_shutdown = True + +def parse_args(): + name = sys.argv[1] + transport_method = sys.argv[2] + store_type = sys.argv[3] + connection_details = sys.argv[4] + db_store_method = sys.argv[5] + + return (transport_method, store.build_store(name, store_type, connection_details, db_store_method)) + +def enqueue_input(stdin, queue): + try: + for line in stdin: + queue.put(line) + stdin.close() + except Exception, e: # something happened, finish up and exit + stdin.close() + is_shutdown = True + +def start_stdin_thread(): + stdin_queue = Queue() + stdin_thread = Thread(target=enqueue_input, args=(sys.stdin, stdin_queue)) + stdin_thread.daemon = True + stdin_thread.start() + return stdin_queue + +def get_all_from_queue(queue): + tmp_str = '' + while not queue.empty(): + try: + tmp_str += queue.get_nowait() + except Empty: + break + return tmp_str + +def handle_request(request, store_class): + if request.req_type == msg_enums.REQUEST_SAVE: + success = store_class.save(request.data[0], request.data[1], request.data_type) + return (success, request.id, '') + elif request.req_type == msg_enums.REQUEST_RESTORE: + success, data = store_class.restore(request.data, request.data_type) + return (success, request.id, data) + else: + return (False, request.id, '') + +def main(): + (transport_method, store_class) = parse_args() + request_factory = StoreRequestFactory(transport_method) + response_factory = StoreResponseFactory(transport_method) + chunk_handler = StoreChunkHandler() + stdin_queue = start_stdin_thread() + + signal.signal(signal.SIGINT, shutdown_handler) + + while True: + if is_shutdown: + if stdin_queue.empty(): + time.sleep(0.5) + if stdin_queue.empty(): + store_class.cleanup() + break + + requests = chunk_handler.handle(get_all_from_queue(stdin_queue)) + requests = map(request_factory.rebuild, requests) + for request in requests: + request = request_factory.rebuild(request) + response = handle_request(request, store_class) + response = response_factory.build(response[0], response[1], response[2]) + sys.stdout.write(chunk_handler.sign(response.serialized)) + + +if __name__ == '__main__': + main() From 79cb5b54647ad309a22bb8be3e894b14cc5bc250 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 8 Jul 2013 17:18:43 -0700 Subject: [PATCH 02/48] added unit tests --- .../serialize/runstate/tronstore/__init__.py | 0 .../runstate/tronstore/parallelstore_test.py | 113 ++++++++ .../runstate/tronstore/store_test.py | 244 ++++++++++++++++++ .../runstate/tronstore/parallelstore.py | 2 +- 4 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 tests/serialize/runstate/tronstore/__init__.py create mode 100644 tests/serialize/runstate/tronstore/parallelstore_test.py create mode 100644 tests/serialize/runstate/tronstore/store_test.py diff --git a/tests/serialize/runstate/tronstore/__init__.py b/tests/serialize/runstate/tronstore/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py new file mode 100644 index 000000000..53ea07de8 --- /dev/null +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -0,0 +1,113 @@ +import contextlib +import mock +from testify import TestCase, run, setup_teardown, assert_equal +from tron.serialize import runstate +from tron.serialize.runstate.tronstore.parallelstore import ParallelStore, ParallelKey +from tron.serialize.runstate.tronstore import msg_enums + + +class ParallelStoreTestCase(TestCase): + + @setup_teardown + def setup_store(self): + self.config = mock.Mock( + name='test_config', + transport_method='pickle', + store_type='shelve', + connection_details=None, + db_store_method=None, + buffer_size=1 + ) + with contextlib.nested( + mock.patch('twisted.internet.reactor.spawnProcess', autospec=True), + mock.patch('twisted.internet.reactor.run', autospec=True), + mock.patch('tron.serialize.runstate.tronstore.parallelstore.StoreProcessProtocol', autospec=True) + ) as (self.spawn_patch, self.run_patch, self.process_patch): + self.store = ParallelStore(self.config) + yield + + def test__init__(self): + self.process_patch.assert_called_once_with(self.store.response_factory) + + def test_start_process(self): + self.spawn_patch.assert_called_once_with( + self.store.process, + "serialize/runstate/tronstore/tronstore", + ["tronstore", + self.config.name, + self.config.transport_method, + self.config.store_type, + self.config.connection_details, + self.config.db_store_method]) + self.run_patch.assert_called_once_with() + + def test_build_key(self): + key_type = runstate.JOB_STATE + key_name = 'the_fun_ends_here' + assert_equal(self.store.build_key(key_type, key_name), ParallelKey(key_type, key_name)) + + def test_save(self): + key_value_pairs = [ + (self.store.build_key(runstate.JOB_STATE, 'riki_the_pubstar'), + {'butterfly': 'time_to_buy_mkb'}), + (self.store.build_key(runstate.JOB_STATE, 'you_died_30_seconds_in'), + {'it_was_lag': 'i_swear'}) + ] + with mock.patch.object(self.store.request_factory, 'build') as build_patch: + self.store.save(key_value_pairs) + for key, state_data in key_value_pairs: + build_patch.assert_any_call(msg_enums.REQUEST_SAVE, key.type, (key.key, state_data)) + assert self.store.process.send_request.called + + def test_restore_single_success(self): + key = self.store.build_key(runstate.JOB_STATE, 'zeus_ult') + fake_response = mock.Mock(data=10, successful=True) + with contextlib.nested( + mock.patch.object(self.store.request_factory, 'build'), + mock.patch.object(self.store.process, 'send_request_get_response', return_value=fake_response), + ) as (build_patch, send_patch): + assert_equal(self.store.restore_single(key), fake_response.data) + build_patch.assert_called_once_with(msg_enums.REQUEST_RESTORE, key.type, key.key) + assert send_patch.called + + def test_restore_single_failure(self): + key = self.store.build_key(runstate.JOB_STATE, 'rip_ryan_davis') + fake_response = mock.Mock(data=777, successful=False) + with contextlib.nested( + mock.patch.object(self.store.request_factory, 'build'), + mock.patch.object(self.store.process, 'send_request_get_response', return_value=fake_response), + ) as (build_patch, send_patch): + assert not self.store.restore_single(key) + build_patch.assert_called_once_with(msg_enums.REQUEST_RESTORE, key.type, key.key) + assert send_patch.called + + def test_restore(self): + keys = [self.store.build_key(runstate.JOB_STATE, 'true_steel'), + self.store.build_key(runstate.JOB_STATE, 'the_test')] + fake_response = mock.Mock() + response_dict = dict((key, fake_response) for key in keys) + with mock.patch.object(self.store, 'restore_single', return_value=fake_response) as restore_patch: + assert_equal(self.store.restore(keys), response_dict) + for key in keys: + restore_patch.assert_any_call(key) + + def test_cleanup(self): + with mock.patch.object(self.store, 'cleanup') as clean_patch: + self.store.cleanup() + clean_patch.assert_called_once_with() + + def test_load_config(self): + new_config = mock.Mock() + with contextlib.nested( + mock.patch.object(self.store.request_factory, 'update_method'), + mock.patch.object(self.store.response_factory, 'update_method'), + mock.patch.object(self.store.process, 'shutdown'), + mock.patch.object(self.store, 'start_process'), + mock.patch('tron.serialize.runstate.tronstore.process.StoreProcessProtocol', autospec=True) + ) as (request_patch, response_patch, shutdown_patch, start_patch, process_patch): + self.store.load_config(new_config) + request_patch.assert_called_once_with(new_config.transport_method) + response_patch.assert_called_once_with(new_config.transport_method) + shutdown_patch.assert_called_once_with() + self.process_patch.assert_any_call(self.store.response_factory) + assert_equal(self.store.config, new_config) diff --git a/tests/serialize/runstate/tronstore/store_test.py b/tests/serialize/runstate/tronstore/store_test.py new file mode 100644 index 000000000..c838c1a22 --- /dev/null +++ b/tests/serialize/runstate/tronstore/store_test.py @@ -0,0 +1,244 @@ +import os +import shelve +import tempfile +from testify import TestCase, run, setup, assert_equal, teardown +from tron.serialize.runstate.tronstore.store import ShelveStore, SQLStore, MongoStore, YamlStore +from tron.serialize.runstate.tronstore.transport import JSONTransport +from tron.serialize import runstate + + +class ShelveStoreTestCase(TestCase): + + @setup + def setup_store(self): + self.filename = os.path.join(tempfile.gettempdir(), 'state') + self.store = ShelveStore(self.filename, None, None) + + @teardown + def teardown_store(self): + os.unlink(self.filename) + self.store.cleanup() + + def test__init__(self): + assert_equal(self.filename, self.store.fname) + + def test_save(self): + data_type = runstate.JOB_STATE + key_value_pairs = [ + ("one", {'some': 'data'}), + ("two", {'its': 'fake'}) + ] + for key, value in key_value_pairs: + self.store.save(key, value, data_type) + self.store.cleanup() + stored = shelve.open(self.filename) + for key, value in key_value_pairs: + assert_equal(stored['(%s__%s)' % (data_type, key)], value) + + def test_restore_success(self): + data_type = runstate.JOB_STATE + keys = ["three", "four"] + value = {'some': 'data'} + store = shelve.open(self.filename) + for key in keys: + store['(%s__%s)' % (data_type, key)] = value + store.close() + + for key in keys: + assert_equal((True, value), self.store.restore(key, data_type)) + + def test_restore_failure(self): + keys = ["nope", "theyre not there"] + for key in keys: + assert_equal((False, None), self.store.restore(key, 'data_type')) + + +class SQLStoreTestCase(TestCase): + + @setup + def setup_store(self): + details = 'sqlite:///:memory:' + self.store = SQLStore('name', details, JSONTransport) + + @teardown + def teardown_store(self): + self.store.cleanup() + + def test_create_engine(self): + assert_equal(self.store.engine.url.database, ':memory:') + + def test_create_tables(self): + assert self.store.job_state_table.name + assert self.store.job_run_table.name + assert self.store.service_table.name + assert self.store.metadata_table.name + + def test_save(self): + data_type = runstate.SERVICE_STATE + key = 'dotes' + state_data = {'the_true_victim_is': 'roshan'} + self.store.save(key, state_data, data_type) + + rows = self.store.engine.execute(self.store.service_table.select()) + assert_equal(rows.fetchone(), ('dotes', self.store.serializer.serialize(state_data))) + + def test_restore_success(self): + data_type = runstate.JOB_STATE + key = '20minbf' + state_data = {'ogre_magi': 'pure_skill'} + + self.store.save(key, state_data, data_type) + assert_equal((True, state_data), self.store.restore(key, data_type)) + + def test_restore_failure(self): + data_type = runstate.JOB_RUN_STATE + key = 'someone_get_gem' + + assert_equal((False, None), self.store.restore(key, data_type)) + + +class MongoStoreTestCase(TestCase): + + store = None + + @setup + def setup_store(self): + import mock + self.db_name = 'test_base' + details = "hostname=localhost&port=5555" + with mock.patch('pymongo.Connection', autospec=True): + self.store = MongoStore(self.db_name, details, None) + + # Since we mocked the pymongo connection, a teardown isn't needed. + + def _create_doc(self, key, doc, data_type): + import pymongo + db = pymongo.Connection()[self.db_name] + doc['_id'] = key + db[self.store.TYPE_TO_COLLECTION_MAP[data_type]].save(doc) + db.connection.disconnect() + + def test__init__(self): + assert_equal(self.store.db_name, self.db_name) + + def test_parse_connection_details(self): + details = "hostname=mongoserver&port=55555" + params = self.store._parse_connection_details(details) + assert_equal(params, {'hostname': 'mongoserver', 'port': '55555'}) + + def test_parse_connection_details_with_user_creds(self): + details = "hostname=mongoserver&port=55555&username=ted&password=sam" + params = self.store._parse_connection_details(details) + expected = { + 'hostname': 'mongoserver', + 'port': '55555', + 'username': 'ted', + 'password': 'sam'} + assert_equal(params, expected) + + def test_parse_connection_details_none(self): + params = self.store._parse_connection_details(None) + assert_equal(params, {}) + + def test_parse_connection_details_empty(self): + params = self.store._parse_connection_details("") + assert_equal(params, {}) + + def test_save(self): + import mock + collection = mock.Mock() + key = 'gotta_have_that_dotes' + state_data = {'skywrath_mage': 'more_like_early_game_page'} + data_type = runstate.JOB_STATE + with mock.patch.object(self.store, 'db', + new={self.store.TYPE_TO_COLLECTION_MAP[data_type]: + collection} + ): + self.store.save(key, state_data, data_type) + state_data['_id'] = key + collection.save.assert_called_once_with(state_data) + + def test_restore_success(self): + import mock + key = 'stop_feeding' + state_data = {'0_and_7': 'only_10_minutes_in'} + data_type = runstate.JOB_RUN_STATE + collection = mock.Mock() + collection.find_one = mock.Mock(return_value=state_data) + with mock.patch.object(self.store, 'db', + new={self.store.TYPE_TO_COLLECTION_MAP[data_type]: + collection} + ): + assert_equal(self.store.restore(key, data_type), (True, state_data)) + collection.find_one.assert_called_once_with(key) + + def test_restore_failure(self): + import mock + key = 'gg_team_fed' + data_type = runstate.SERVICE_STATE + collection = mock.Mock() + collection.find_one = mock.Mock(return_value=None) + with mock.patch.object(self.store, 'db', + new={self.store.TYPE_TO_COLLECTION_MAP[data_type]: + collection} + ): + assert_equal(self.store.restore(key, data_type), (False, None)) + collection.find_one.assert_called_once_with(key) + + +class YamlStoreTestCase(TestCase): + + @setup + def setup_store(self): + self.filename = os.path.join(tempfile.gettempdir(), 'yaml_state') + self.store = YamlStore(self.filename, None, None) + self.test_data = { + self.store.TYPE_MAPPING[runstate.JOB_STATE]: {'a': 1}, + self.store.TYPE_MAPPING[runstate.JOB_RUN_STATE]: {'b': 2}, + self.store.TYPE_MAPPING[runstate.SERVICE_STATE]: {'c': 3} + } + + @teardown + def teardown_store(self): + try: + os.unlink(self.filename) + except OSError: + pass + + def test_restore_success(self): + import yaml + with open(self.filename, 'w') as fh: + yaml.dump(self.test_data, fh) + self.store = YamlStore(self.filename, None, None) + + data_types = [runstate.JOB_STATE, runstate.JOB_RUN_STATE, runstate.SERVICE_STATE] + for data_type in data_types: + for key in self.test_data[self.store.TYPE_MAPPING[data_type]].keys(): + success, value = self.store.restore(key, data_type) + assert success + assert_equal(self.test_data[self.store.TYPE_MAPPING[data_type]][key], value) + + def test_restore_failure(self): + assert_equal(self.store.restore('gg_stick_pro_build', runstate.JOB_STATE), (False, None)) + + def test_save(self): + import yaml + job_data = {'euls_on_sk': 'sounds_legit'} + run_data = {'phantom_cancer': 'needs_diffusal_level_2'} + service_data = {'everyone_go_dagon': 'hey_look_we_won'} + expected = { + self.store.TYPE_MAPPING[runstate.JOB_STATE]: job_data, + self.store.TYPE_MAPPING[runstate.JOB_RUN_STATE]: run_data, + self.store.TYPE_MAPPING[runstate.SERVICE_STATE]: service_data, + } + self.store.save(job_data.keys()[0], job_data.values()[0], runstate.JOB_STATE) + self.store.save(run_data.keys()[0], run_data.values()[0], runstate.JOB_RUN_STATE) + self.store.save(service_data.keys()[0], service_data.values()[0], runstate.SERVICE_STATE) + + assert_equal(self.store.buffer, expected) + with open(self.filename, 'r') as fh: + actual = yaml.load(fh) + assert_equal(actual, expected) + +if __name__ == "__main__": + run() diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index b5d073c12..ef5609e21 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -82,7 +82,7 @@ def cleanup(self): # PersistentStateManager whenever the config is updated, which removes # the need for changing config related things here (since a new instance # of this class will be created anyway). - def config(self, new_config): + def load_config(self, new_config): """Reconfigure the storing mechanism to use a new configuration.""" self.config = new_config self.request_factory.update_method(new_config.transport_method) From d0e4dc78f24f2ac66010cc7e4e1a74ac67434b5e Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 9 Jul 2013 11:53:46 -0700 Subject: [PATCH 03/48] unit tests done --- .../runstate/tronstore/parallelstore_test.py | 1 + .../runstate/tronstore/process_test.py | 148 ++++++++++++++++++ tron/serialize/runstate/tronstore/chunking.py | 12 +- tron/serialize/runstate/tronstore/process.py | 10 +- 4 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 tests/serialize/runstate/tronstore/process_test.py diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py index 53ea07de8..37818c10f 100644 --- a/tests/serialize/runstate/tronstore/parallelstore_test.py +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -110,4 +110,5 @@ def test_load_config(self): response_patch.assert_called_once_with(new_config.transport_method) shutdown_patch.assert_called_once_with() self.process_patch.assert_any_call(self.store.response_factory) + start_patch.assert_called_once_with() assert_equal(self.store.config, new_config) diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py new file mode 100644 index 000000000..64a71be1e --- /dev/null +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -0,0 +1,148 @@ +import contextlib +import mock +from testify import TestCase, run, assert_equal, assert_raises, setup_teardown +from tron.serialize.runstate.tronstore.process import StoreProcessProtocol, TronStoreError +from tron.serialize.runstate.tronstore import chunking + +class StoreProcessProtocolTestCase(TestCase): + + @setup_teardown + def setup_process(self): + with mock.patch('twisted.internet.reactor.stop', autospec=True) as self.stop_patch: + self.factory = mock.Mock() + self.process = StoreProcessProtocol(self.factory) + yield + + def test__init__(self): + assert_equal(self.process.response_factory, self.factory) + assert_equal(self.process.requests, {}) + assert_equal(self.process.responses, {}) + assert_equal(self.process.semaphores, {}) + assert not self.process.is_shutdown + + def test_outRecieved_chunked(self): + data = 'deadly_premonition_br' + self.process.outRecieved(data) + assert_equal(self.process.chunker.chunk, data) + + def test_outRecieved_full_no_monitor(self): + data = 'no_one_str_should_have_all_that_power' + chunking.CHUNK_SIGNING_STR + fake_id = 77 + fake_response = mock.Mock(id=fake_id, success=True) + self.factory.rebuild = mock.Mock(return_value=fake_response) + self.process.requests[fake_id] = fake_response + self.process.outRecieved(data) + assert_equal(self.process.chunker.chunk, '') + assert_equal(self.process.responses, {}) + assert_equal(self.process.requests, {}) + assert_equal(self.process.semaphores, {}) + + def test_outRecieved_full_with_monitor(self): + data = 'throwing_dark' + chunking.CHUNK_SIGNING_STR + fake_id = 77 + fake_response = mock.Mock(id=fake_id, success=True) + self.factory.rebuild = mock.Mock(return_value=fake_response) + fake_semaphore = mock.Mock() + self.process.semaphores[fake_id] = fake_semaphore + self.process.requests[fake_id] = fake_response + self.process.outRecieved(data) + assert_equal(self.process.chunker.chunk, '') + assert_equal(self.process.responses, {fake_id: fake_response}) + assert_equal(self.process.semaphores, {fake_id: fake_semaphore}) + assert_equal(self.process.requests, {}) + fake_semaphore.release.assert_called_once_with() + + def test_processExited_running(self): + self.process.is_shutdown = False + assert_raises(TronStoreError, self.process.processExited, mock.Mock(getErrorMessage=lambda: 'test')) + + def test_processExited_shutdown(self): + self.process.is_shutdown = True + self.process.processExited('its_a_website') + + def test_processEnded_running(self): + self.process.is_shutdown = False + with mock.patch.object(self.process, 'transport') as lose_patch: + self.process.processEnded('about_videogames') + lose_patch.loseConnection.assert_called_once_with() + self.stop_patch.assert_called_once_with() + + def test_processEnded_shutdown(self): + self.process.is_shutdown = True + with mock.patch.object(self.process, 'transport') as lose_patch: + self.process.processEnded('this_aint_no_game') + assert not self.stop_patch.called + assert not lose_patch.loseConnection.called + + def test_send_request_running(self): + self.process.is_shutdown = False + fake_id = 77 + test_request = mock.Mock(serialized='sunny_sausalito', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process, 'transport'), + mock.patch.object(self.process.chunker, 'sign') + ) as (trans_patch, sign_patch): + self.process.send_request(test_request) + assert_equal(self.process.requests[fake_id], test_request) + sign_patch.assert_called_once_with(test_request.serialized) + trans_patch.write.assert_called_once_with(self.process.chunker.sign(test_request.serialized)) + + def test_send_request_shutdown(self): + self.process.is_shutdown = True + fake_id = 77 + test_request = mock.Mock(serialized='whiskey_media', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process, 'transport'), + mock.patch.object(self.process.chunker, 'sign') + ) as (trans_patch, sign_patch): + self.process.send_request(test_request) + assert_equal(self.process.requests, {}) + assert not sign_patch.called + assert not trans_patch.write.called + + def test_send_request_get_response_running(self): + self.process.is_shutdown = False + fake_id = 77 + test_request = mock.Mock(serialized='objection', id=fake_id) + test_response = mock.Mock(id=fake_id, data='overruled', success=True) + self.process.responses[fake_id] = test_response + with contextlib.nested( + mock.patch.object(self.process, 'transport'), + mock.patch.object(self.process.chunker, 'sign'), + mock.patch('tron.serialize.runstate.tronstore.process.Semaphore', autospec=True) + ) as (trans_patch, sign_patch, sema_patch): + assert_equal(self.process.send_request_get_response(test_request), test_response.data) + assert_equal(self.process.requests, {fake_id: test_request}) + assert_equal(self.process.semaphores, {}) + assert_equal(self.process.responses, {}) + sign_patch.assert_called_once_with(test_request.serialized) + trans_patch.write.assert_called_once_with(self.process.chunker.sign(test_request.serialized)) + sema_patch.assert_called_once_with(0) + + def test_send_request_get_response_shutdown(self): + self.process.is_shutdown = True + fake_id = 77 + test_request = mock.Mock(serialized='does_he_look_like_a', id=fake_id) + test_response = mock.Mock(id=fake_id, data='what', success=True) + self.process.responses[fake_id] = test_response + with contextlib.nested( + mock.patch.object(self.process, 'transport'), + mock.patch.object(self.process.chunker, 'sign'), + mock.patch('tron.serialize.runstate.tronstore.process.Semaphore', autospec=True) + ) as (trans_patch, sign_patch, sema_patch): + assert_equal(self.process.send_request_get_response(test_request), None) + assert_equal(self.process.requests, {}) + assert_equal(self.process.semaphores, {}) + assert_equal(self.process.responses, {fake_id: test_response}) + assert not sign_patch.called + assert not trans_patch.write.called + assert not sema_patch.called + + def test_shutdown(self): + self.process.is_shutdown = False + with mock.patch.object(self.process, 'transport') as trans_patch: + self.process.shutdown() + assert self.process.is_shutdown + trans_patch.signalProcess.assert_called_once_with('INT') + trans_patch.loseConnection.assert_called_once_with() + self.stop_patch.assert_called_once_with() diff --git a/tron/serialize/runstate/tronstore/chunking.py b/tron/serialize/runstate/tronstore/chunking.py index fc74418d0..0c1efe7a1 100644 --- a/tron/serialize/runstate/tronstore/chunking.py +++ b/tron/serialize/runstate/tronstore/chunking.py @@ -11,9 +11,11 @@ def sign(self, data): def handle(self, data): self.chunk += data chunks = self.chunk.split(CHUNK_SIGNING_STR) - # if the chunk doesn't end with the signed string, the message was - # incomplete and should be saved for the next time .handle is called. - if not self.chunk.endswith(CHUNK_SIGNING_STR): - self.chunk = chunks[-1:] - chunks = chunks[:-1] + # split actually has this nice behavior where it makes the last element + # of the list of split strings an empty string if the original string + # ended with the sequence that was used to split with. This allows + # a nice, simple way to either get any remaining characters, or + # simply setting the chunk to '' again. + self.chunk = chunks[-1] if chunks else '' + chunks = chunks[:-1] return chunks diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index e0e577dee..0c309329e 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -41,7 +41,7 @@ def outRecieved(self, data): if response.id in self.requests: if not response.success: log.warn("tronstore request #%d failed. Request type was %d." % (response.id, self.requests[response.id].req_type)) - if response.id in self.monitors: + if response.id in self.semaphores: self.responses[response.id] = response self.semaphores[response.id].release() del self.requests[response.id] @@ -77,7 +77,7 @@ def send_request_get_response(self, request): self.semaphores[request.id].acquire() del self.semaphores[request.id] response = self.responses[request.id] - del self.responses[requests.id] + del self.responses[request.id] return response.data if response.success else None def shutdown(self): @@ -91,8 +91,8 @@ def shutdown(self): self.is_shutdown = True time_waited = 0 while (not len(self.requests.items()) == 0) and time_waited < self.SHUTDOWN_TIMEOUT: - time.sleep(SHUTDOWN_SLEEP) # wait for all pending requests to finish - time_waited += SHUTDOWN_SLEEP + time.sleep(self.SHUTDOWN_SLEEP) # wait for all pending requests to finish + time_waited += self.SHUTDOWN_SLEEP self.transport.signalProcess("INT") self.transport.loseConnection() - self.reactor.stop() + reactor.stop() From 17f0a21e84def7b41c900516343c89142938721d Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 9 Jul 2013 15:01:57 -0700 Subject: [PATCH 04/48] docstrings. docstrings everywhere --- tron/serialize/runstate/tronstore/chunking.py | 13 ++++++- tron/serialize/runstate/tronstore/messages.py | 39 ++++++++++++++++--- .../runstate/tronstore/parallelstore.py | 12 ++++-- tron/serialize/runstate/tronstore/process.py | 28 +++++++++++-- tron/serialize/runstate/tronstore/store.py | 2 +- .../serialize/runstate/tronstore/transport.py | 7 ++++ tron/serialize/runstate/tronstore/tronstore | 28 +++++++++++++ 7 files changed, 113 insertions(+), 16 deletions(-) diff --git a/tron/serialize/runstate/tronstore/chunking.py b/tron/serialize/runstate/tronstore/chunking.py index 0c1efe7a1..626756d61 100644 --- a/tron/serialize/runstate/tronstore/chunking.py +++ b/tron/serialize/runstate/tronstore/chunking.py @@ -1,21 +1,30 @@ CHUNK_SIGNING_STR = '\xDE\xAD\xBE\xEF\x00' class StoreChunkHandler(object): + """A simple chunk handler for dealing with string based I/O stream + messaging. Works by one end using the sign() function to sign the + serialized string and then using handle() on the opposite end of the wire. + + This is used by tronstore, as the twisted stdin/out handler chunks.""" def __init__(self): self.chunk = '' def sign(self, data): + """Sign a string to be sent.""" return (data + CHUNK_SIGNING_STR) def handle(self, data): + """Handle a signed string, returning all individual strings that were + signed as a list. + """ self.chunk += data chunks = self.chunk.split(CHUNK_SIGNING_STR) # split actually has this nice behavior where it makes the last element - # of the list of split strings an empty string if the original string + # of the returned list an empty string if the original string # ended with the sequence that was used to split with. This allows # a nice, simple way to either get any remaining characters, or - # simply setting the chunk to '' again. + # simply setting the chunk to '' again, without any extra code. self.chunk = chunks[-1] if chunks else '' chunks = chunks[:-1] return chunks diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 4a5821ce7..0cfd913f9 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -6,17 +6,25 @@ 'msgpack': MsgPackTransport, 'yaml': YamlTransport } - +# a simple max integer to prevent ids from growing indefinitely MAX_MSG_ID = 2**32 - 1 class StoreRequestFactory(object): + """A factory to generate requests that need to be converted to serialized + strings and back. All the factory itself does is keep track of what + serialization method was set by the configuration, and then constructs + specific StoreRequest objects using that method. + """ def __init__(self, method): self.serializer = transport_class_map[method] self.id_counter = 1 def _increment_counter(self): + """A simple method to make sure that we don't indefinitely increase + the id assigned to StoreRequests. + """ return self.id+1 if not self.id == MAX_MSG_ID else 0 def build(self, req_type, data_type, data): @@ -36,6 +44,11 @@ def get_method(self): class StoreResponseFactory(object): + """A factory to generate responses that need to be converted to serialized + strings and back. The factory itself just keeps track of what serialization + method was specified by the configuration, and then constructs specific + StoreResponse objects using that method. + """ def __init__(self, method): self.serializer = transport_class_map[method] @@ -56,13 +69,21 @@ def get_method(self): class StoreRequest(object): + """An object representing a request to tronstore. The request has four + essential attributes: + id - an integer identifier, used for matching requests with responses + req_type - the request type from msg_enums.py, such as save/restore + data_type - the type of data the request is for. there are four kinds + of saved state_data: job, jobrun, service, and meta state_data. + data - the data required for the request, like a name or state_data + """ def __init__(self, req_id, req_type, data_type, data, method): - self.id = req_id - self.req_type = req_type - self.data = data - self.data_type = data_type - self.method = method + self.id = req_id + self.req_type = req_type + self.data = data + self.data_type = data_type + self.method = method self.serialized = self.get_serialized() @classmethod @@ -80,6 +101,12 @@ def get_serialized(self): class StoreResponse(object): + """An object representing a response from tronstore. The response has three + essential attributes: + id - matches the id of some request so this can be matched with it + success - shows if the request matching this response was successful + data - data requested by a request, if any + """ def __init__(self, req_id, success, data, method): self.id = req_id diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index ef5609e21..0057f62fc 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -31,12 +31,12 @@ def __hash__(self): return hash(self.key) class ParallelStore(object): - """Persist state using a paralleled storing mechanism. This uses + """Persist state using a paralleled storing mechanism, tronstore. This uses the Twisted library to run the tronstore executable in a separate process, and handles all communication between trond and tronstore. This class handles construction of all messages that need to be sent - to tronstore.""" + to tronstore based on requests given by the MCP.""" def __init__(self, config): self.config = config @@ -46,6 +46,11 @@ def __init__(self, config): self.start_process() def start_process(self): + """Use twisted to spawn the tronstore process. + + The command line arguments given to spawnProcess are in a + HARDCODED ORDER that MUST match the order that tronstore parses them. + """ reactor.spawnProcess(self.process, "serialize/runstate/tronstore/tronstore", ["tronstore", self.config.name, @@ -83,7 +88,8 @@ def cleanup(self): # the need for changing config related things here (since a new instance # of this class will be created anyway). def load_config(self, new_config): - """Reconfigure the storing mechanism to use a new configuration.""" + """Reconfigure the storing mechanism to use a new configuration + by shutting down and restarting tronstore.""" self.config = new_config self.request_factory.update_method(new_config.transport_method) self.process.shutdown() diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index 0c309329e..f9d02acd7 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -17,10 +17,13 @@ def __str__(self): class StoreProcessProtocol(ProcessProtocol): """The class that actually communicates with tronstore. This is a subclass - of the twisted ProcessProtocol class, which can run and asynchronously - communicate with a child proccess. The requests and responses are matched - together by the unique integer ids assigned to each request (which are also - present in responses). + of the twisted ProcessProtocol class, which has a set of internals that can + communicate with a child proccess via stdin/stdout via interrupts. + + Because of this I/O structure imposed by twisted, there are two types of + messages: requests and responses. Responses are always of the same form, + while requests have an enumerator (see msg_enums.py) to identify the + type of request. """ SHUTDOWN_TIMEOUT = 5.0 @@ -35,9 +38,21 @@ def __init__(self, response_factory): self.is_shutdown = False def outRecieved(self, data): + """Called via interrupt whenever twisted sees something written by the + process into stdout, where data is whatever the process wrote. + Since the only thing written to stdout are serialized responses from + tronstore, this method deals with matching the response to the + appropriate request if the request needed it. + + As some requests actually require a response (see: restore requests), + this method also wakes up the main trond thread if it was blocking on a + response from tronstore. + """ responses = self.chunker.handle(data) for response_str in responses: response = self.response_factory.rebuild(response_str) + # Requests that don't actually require a response don't put + # themselves inside of the requests dict. if response.id in self.requests: if not response.success: log.warn("tronstore request #%d failed. Request type was %d." % (response.id, self.requests[response.id].req_type)) @@ -47,10 +62,15 @@ def outRecieved(self, data): del self.requests[response.id] def processExited(self, reason): + """Called by twisted whenever the process exits. + If the process didn't exit cleanly (we didn't shut it down), + then we need to raise an exception. + """ if not self.is_shutdown: raise TronStoreError(reason.getErrorMessage()) def processEnded(self, reason): + """Called by twisted whenever the process ends. Cleans up.""" if not self.is_shutdown: self.transport.loseConnection() reactor.stop() diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 0ea2c7540..4229ce1fd 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -32,7 +32,7 @@ def __repr__(self): class SQLStore(object): - """Store state using SQLAlchemy.""" + """Store state using SQLAlchemy. Creates tables if needed.""" def __init__(self, name, connection_details, serializer): import sqlalchemy as sql diff --git a/tron/serialize/runstate/tronstore/transport.py b/tron/serialize/runstate/tronstore/transport.py index d8fb80007..b8f174ec6 100644 --- a/tron/serialize/runstate/tronstore/transport.py +++ b/tron/serialize/runstate/tronstore/transport.py @@ -1,3 +1,10 @@ +"""Message transport modules for tronstore. This allows for simple writing +of stdin/out with strings that can then be put back into tuples of data +for rebuilding messages. + +This is also used by the SQLAlchemy store object, an option for saving state +with tronstore, by serializing the state data into a string that's saved in +a SQL database, or by deserializing strings that are saved into state data.""" import simplejson as json import cPickle as pickle diff --git a/tron/serialize/runstate/tronstore/tronstore b/tron/serialize/runstate/tronstore/tronstore index 6fe8af914..54769d1a4 100755 --- a/tron/serialize/runstate/tronstore/tronstore +++ b/tron/serialize/runstate/tronstore/tronstore @@ -1,4 +1,19 @@ #!/usr/bin/env python +"""This process is spawned by trond in order to offload state save/load +operations such that trond can focus on the more important things without +blocking for large chunks of time. It takes command line arguments (the order +of which is specifically hardcoded) in order to configure itself and use +the correct methods for state saving and message transport with trond. + +Messages are sent via stdin/out, as the twisted framework that provides +management of child processes uses it for communication. Because the core +Python stdin reader is blocking, a separate thread is spawned in tronstore +to buffer all messages that come via stdin. + +The process gracefully shuts down whenever it recieves a SIGINT by first +processing all requests it has already read from stdin. trond capitalizes on +this behavior by propagating any SIGINT signals it receives to tronstore. +""" import sys import os import signal @@ -14,9 +29,13 @@ from tron.serialize.tronstore.chunking import StoreChunkHandler is_shutdown = False def shutdown_handler(): + """Signal the process to shut down, finishing any requests it already had.""" is_shutdown = True def parse_args(): + """Parse the command line arguments. THIS MUST MATCH THE ORDER GIVEN BY + parallelstore.py when it initializes the tronstore process. + """ name = sys.argv[1] transport_method = sys.argv[2] store_type = sys.argv[3] @@ -26,6 +45,8 @@ def parse_args(): return (transport_method, store.build_store(name, store_type, connection_details, db_store_method)) def enqueue_input(stdin, queue): + """Enqueue anything read by stdin into a queue. This is run in a separate + thread such that requests can be processed without blocking.""" try: for line in stdin: queue.put(line) @@ -35,6 +56,7 @@ def enqueue_input(stdin, queue): is_shutdown = True def start_stdin_thread(): + """Starts the thread that will enqueue all stdin data into the stdin_queue.""" stdin_queue = Queue() stdin_thread = Thread(target=enqueue_input, args=(sys.stdin, stdin_queue)) stdin_thread.daemon = True @@ -42,6 +64,7 @@ def start_stdin_thread(): return stdin_queue def get_all_from_queue(queue): + """Gets all of the requests from the stdin_queue, returning one long string.""" tmp_str = '' while not queue.empty(): try: @@ -51,6 +74,7 @@ def get_all_from_queue(queue): return tmp_str def handle_request(request, store_class): + """Handle a request by acting on store_class with the appropriate action.""" if request.req_type == msg_enums.REQUEST_SAVE: success = store_class.save(request.data[0], request.data[1], request.data_type) return (success, request.id, '') @@ -61,6 +85,10 @@ def handle_request(request, store_class): return (False, request.id, '') def main(): + """The main run loop for tronstore. This loop sets up everything + based on the command line arguments tronstore got, and then simply + waits for requests to handle from stdin, and writes responses to + stdout. The process will run until an error or SIGINT occurs.""" (transport_method, store_class) = parse_args() request_factory = StoreRequestFactory(transport_method) response_factory = StoreResponseFactory(transport_method) From 53b3b46c09d7d05a402b5a6def961e55f1f8f6f1 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 9 Jul 2013 17:24:29 -0700 Subject: [PATCH 05/48] made transport_method an optional variable --- tron/config/config_parse.py | 1 + tron/config/schema.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 3f08b7752..640e5471c 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -393,6 +393,7 @@ class ValidateStatePersistence(Validator): 'buffer_size': 1, 'connection_details': None, 'db_store_method': 'msgpack', + 'transport_method': 'pickle', } validators = { diff --git a/tron/config/schema.py b/tron/config/schema.py index f65b76726..ccaeea50d 100644 --- a/tron/config/schema.py +++ b/tron/config/schema.py @@ -90,11 +90,11 @@ def config_object_factory(name, required=None, optional=None): [ 'name', 'store_type', - 'transport_method', ],[ 'connection_details', 'buffer_size', 'db_store_method', + 'transport_method', ]) From ddc37af4707d5c388df359439deb2093ae448ee3 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 10 Jul 2013 15:33:44 -0700 Subject: [PATCH 06/48] first steps for state migration --- .../migrate_state_from_0.6.1_to_0.6.2.py | 177 ++++++++++++++++++ tron/config/config_parse.py | 8 +- 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 tools/migration/migrate_state_from_0.6.1_to_0.6.2.py diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py new file mode 100644 index 000000000..d922d36d0 --- /dev/null +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -0,0 +1,177 @@ +"""Usage: %prog [options] + +This is a script to convert old state storing containers into the new +objects used by Tron v0.6.2 and tronstore. The script will use the same +mechanism for storing state as specified in the Tron configuration file +(except for SQL, which will default to using simplejson for serializing +state data into its database). This can be overriden via command line +options, which allow for full configuration of the mechanisms used to store +the new state objects. + +Please upgrade to Tron v0.6.2 before running this script. Also note that +migrate_state.py will NOT work until running this script, as it has been +changed to work with v0.6.2's version of state storing. + +Make sure that the working dir is the same as the one used in your Tron +configuration! Otherwise, this script won't be able to load the config file +and make the magic happen. + +***IMPORTANT*** +When using SQLAlchemy/MongoDB storing mechanisms, the -c option for setting +NEW connection detail parameters MUST be set. Because we don't want to clobber +the old data, this script requires that there is a new database for saving +the new state data, meaning new connection parameters. This script +doesn't check rigorously that the new connection details given are valid; +however, it does verify that the details aren't exactly the same. + + +Command line options: + -c str Set new connection details to str for SQL/MongoDB storing. This + is REQUIRED for using SQL/MongoDB as the new state store. + + -m Set a new mechanism for storing the new state objects. + Defaults to whatever store_type was set to in the Tron + configuration file. + Options are sql, mongo, yaml, and shelve. + + -t Set a new method for transporting the new state objects to + tronstore. Defaults to whatever was set to transport_method in the + Tron configuration file, or pickle if it isn't set. + Options are pickle, yaml, msgpack, and json. + + -d Set a new method for storing state data within an SQL database. + Defaults to whatever was set to db_store_method in the Tron + configuration file, or json if it isn't set. Only used if + SQLAlchemy is the storing mechanism. + Options are pickle, yaml, msgpack, and json. +""" + +import sys +import os +import copy + +from tron.commands import cmd_utils +from tron.config import ConfigError +from tron.config.schema import StatePersistenceTypes, StateTransportTypes +from tron.config.manager import ConfigManager +import tron.serialize.runstate +from tron.serialize.runstate.shelvestore import ShelveStateStore +from tron.serialize.runstate.mongostore import MongoStateStore +from tron.serialize.runstate.yamlstore import YamlStateStore +from tron.serialize.runstate.sqlalchemystore import SQLAlchemyStateStore +from tron.serialize.runstate.tronstore.parallelstore import ParallelStore +from tron.serialize.runstate.statemanager import StateMetadata + +def parse_options(): + usage = "usage: %prog [options] " + parser = cmd_utils.build_option_parser(usage) + parser.add_option("-c", type="string", + help="Set new connection details for db connections", + dest="new_connection_details", default=None) + parser.add_option("-m", type="string", + help="Set new state storing mechanism (store_type)", + dest="store_method", default=None) + parser.add_option("-t", type="string", + help="Set new transport method", + dest="transport_method", default=None) + parser.add_option("-d", type="string", + help="Set new SQL db serialization method (db_store_method)", + dest="db_store_method", default=None) + options, args = parser.parse_args(sys.argv) + return options, args[1], args[2] + +def parse_config(working_dir): + # This shouldn't happen, but hey, sanity checks never hurt anyone + if working_dir.endswith('/'): + conf_dir = working_dir + "config" + else: + conf_dir = working_dir + "/config" + + manager = ConfigManager(conf_dir) + return manager.load() + +def get_old_state_store(state_info): + name = state_info.name + connection_details = state_info.connection_details + store_type = state_info.store_type + + if store_type == StatePersistenceTypes.shelve: + return ShelveStateStore(name) + + if store_type == StatePersistenceTypes.sql: + return SQLAlchemyStateStore(name, connection_details) + + if store_type == StatePersistenceTypes.mongo: + return MongoStateStore(name, connection_details) + + if store_type == StatePersistenceTypes.yaml: + return YamlStateStore(name) + +def compile_new_info(options, state_info, new_file): + new_state_info = copy.deepcopy(state_info) + + new_state_info.name = new_file + + if options.store_method: + new_state_info.store_method = options.store_method + + if options.transport_method: + new_state_info.transport_method = options.transport_method + + if options.db_store_method: + new_state_info.db_store_method = options.db_store_method + + if options.new_connection_details \ + and options.new_connection_details != state_info.connection_details: + new_state_info.connection_details = options.new_connection_details + elif new_state_info.store_type in ('sql', 'mongo'): + raise ConfigError('Must specify new connection_details using -c to use %s' + % new_state_info.store_type) + + return new_state_info + +def copy_metadata(old_store, new_store): + meta_key_old = old_store.build_key(runstate.MCP_STATE, StateMetadata.name) + old_metadata = old_store.restore([meta_key_old])[meta_key_old] + meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) + new_store.save([(meta_key_new, old_metadata)]) + +def copy_services(old_store, new_store, service_names): + for service in service_names: + service_key_old = old_store.build_key(runstate.SERVICE_STATE, service) + old_service_data = old_store.restore([service_key_old])[service_key_old] + service_key_new = new_store.build_key(runstate.SERVICE_STATE, service) + new_store.save([(service_key_new, old_service_data)]) + +def copy_jobs(old_store, new_store, job_names): + for job in job_names: + job_key_old = old_store.build_key(runstate.JOB_STATE, job) + old_job_data = old_store.restore([job_key_old])[job_key_old] + job_state_key = new_store.build_key(runstate.JOB_STATE, job) + + run_ids = [] + for job_run in old_job_data['runs']: + run_ids.append(job_run.run_num) + job_run_key = new_store.build_key(runstate.JOB_RUN_STATE, + job + ('.%s' % job_run.run_num)) + new_store.save([(job_run_key, job_run)]) + + run_ids = sorted(run_ids, reverse=True) + job_state_data = {'enabled': old_job_data['enabled'], 'run_ids': run_ids} + new_store.save([(job_state_key, job_state_data)]) + + +def main(): + (options, working_dir, new_fname) = parse_options() + config = parse_config(working_dir) + state_info = config.get_master().state_persistence + old_store = get_old_state_store(state_info) + new_state_info = compile_new_info(options, state_info, new_fname) + new_store = ParallelStore(new_state_info) + + copy_metadata(old_store, new_store) + copy_services(old_store, new_store, config.get_services().keys()) + copy_jobs(old_store, new_store, config.get_jobs().keys()) + +if __name__ == "__main__": + main() diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 640e5471c..9acd03a41 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -392,7 +392,13 @@ class ValidateStatePersistence(Validator): defaults = { 'buffer_size': 1, 'connection_details': None, - 'db_store_method': 'msgpack', + # This is a tricky one. MessagePack isn't a default python library, + # so it feels a bit wrong to make it a configuration default. However, + # Yaml's terrible and simplejson is slower than both msgpack and + # pickle. Pickle MIGHT be an okay default, but I'm nervous about + # defaulting people into using a Turing Complete serialization method + # for SQL storing. + 'db_store_method': 'json', 'transport_method': 'pickle', } From f1dde3813f6e6af8a8169e5acaaf18f9364ea474 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 10 Jul 2013 15:55:23 -0700 Subject: [PATCH 07/48] more migration --- .../migrate_state_from_0.6.1_to_0.6.2.py | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index d922d36d0..e963bfc7d 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -29,21 +29,24 @@ -c str Set new connection details to str for SQL/MongoDB storing. This is REQUIRED for using SQL/MongoDB as the new state store. - -m Set a new mechanism for storing the new state objects. + -m str Set a new mechanism for storing the new state objects. Defaults to whatever store_type was set to in the Tron configuration file. - Options are sql, mongo, yaml, and shelve. + Options for str are sql, mongo, yaml, and shelve. - -t Set a new method for transporting the new state objects to + -t str Set a new method for transporting the new state objects to tronstore. Defaults to whatever was set to transport_method in the Tron configuration file, or pickle if it isn't set. - Options are pickle, yaml, msgpack, and json. + Options for str are pickle, yaml, msgpack, and json. - -d Set a new method for storing state data within an SQL database. + -d str Set a new method for storing state data within an SQL database. Defaults to whatever was set to db_store_method in the Tron configuration file, or json if it isn't set. Only used if SQLAlchemy is the storing mechanism. - Options are pickle, yaml, msgpack, and json. + Options for str are pickle, yaml, msgpack, and json. + + -f str Set the path for the configuration file to str. This defaults to + /config """ import sys @@ -77,17 +80,17 @@ def parse_options(): parser.add_option("-d", type="string", help="Set new SQL db serialization method (db_store_method)", dest="db_store_method", default=None) + parser.add_option("-f", type="string", + help="Set path to Tron configuration file", + dest="conf_dir", default=None) options, args = parser.parse_args(sys.argv) return options, args[1], args[2] -def parse_config(working_dir): - # This shouldn't happen, but hey, sanity checks never hurt anyone - if working_dir.endswith('/'): - conf_dir = working_dir + "config" +def parse_config(conf_dir): + if conf_dir: + manager = ConfigManager(conf_dir) else: - conf_dir = working_dir + "/config" - - manager = ConfigManager(conf_dir) + manager = ConfigManager('config') return manager.load() def get_old_state_store(state_info): @@ -163,7 +166,8 @@ def copy_jobs(old_store, new_store, job_names): def main(): (options, working_dir, new_fname) = parse_options() - config = parse_config(working_dir) + os.chdir(working_dir) + config = parse_config(options.conf_dir) state_info = config.get_master().state_persistence old_store = get_old_state_store(state_info) new_state_info = compile_new_info(options, state_info, new_fname) From ccf1c36f4f757a1fb09523b315c88c8ebf6c951c Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 11 Jul 2013 10:35:23 -0700 Subject: [PATCH 08/48] conversion script progress, bugfixing --- .../migrate_state_from_0.6.1_to_0.6.2.py | 62 ++++++++++++------- tron/serialize/runstate/tronstore/messages.py | 4 +- .../runstate/tronstore/parallelstore.py | 27 +++++++- tron/serialize/runstate/tronstore/process.py | 2 +- tron/serialize/runstate/tronstore/tronstore | 13 ++-- 5 files changed, 74 insertions(+), 34 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index e963bfc7d..f47323931 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -57,7 +57,7 @@ from tron.config import ConfigError from tron.config.schema import StatePersistenceTypes, StateTransportTypes from tron.config.manager import ConfigManager -import tron.serialize.runstate +from tron.serialize import runstate from tron.serialize.runstate.shelvestore import ShelveStateStore from tron.serialize.runstate.mongostore import MongoStateStore from tron.serialize.runstate.yamlstore import YamlStateStore @@ -113,20 +113,20 @@ def get_old_state_store(state_info): def compile_new_info(options, state_info, new_file): new_state_info = copy.deepcopy(state_info) - new_state_info.name = new_file + new_state_info._replace(name=new_file) if options.store_method: - new_state_info.store_method = options.store_method + new_state_info._replace(store_method=options.store_method) if options.transport_method: - new_state_info.transport_method = options.transport_method + new_state_info._replace(transport_method=options.transport_method) if options.db_store_method: - new_state_info.db_store_method = options.db_store_method + new_state_info._replace(db_store_method=options.db_store_method) if options.new_connection_details \ and options.new_connection_details != state_info.connection_details: - new_state_info.connection_details = options.new_connection_details + new_state_info._replace(connection_details=options.new_connection_details) elif new_state_info.store_type in ('sql', 'mongo'): raise ConfigError('Must specify new connection_details using -c to use %s' % new_state_info.store_type) @@ -135,47 +135,63 @@ def compile_new_info(options, state_info, new_file): def copy_metadata(old_store, new_store): meta_key_old = old_store.build_key(runstate.MCP_STATE, StateMetadata.name) - old_metadata = old_store.restore([meta_key_old])[meta_key_old] - meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) - new_store.save([(meta_key_new, old_metadata)]) + old_metadata_dict = old_store.restore([meta_key_old]) + if old_metadata_dict: + old_metadata = old_metadata_dict[meta_key_old] + meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) + new_store.save([(meta_key_new, old_metadata)]) + assert old_metadata == new_store.restore([meta_key_new]) def copy_services(old_store, new_store, service_names): for service in service_names: service_key_old = old_store.build_key(runstate.SERVICE_STATE, service) - old_service_data = old_store.restore([service_key_old])[service_key_old] - service_key_new = new_store.build_key(runstate.SERVICE_STATE, service) - new_store.save([(service_key_new, old_service_data)]) + import ipdb; ipdb.set_trace() + old_service_dict = old_store.restore([service_key_old]) + if old_service_dict: + old_service_data = old_service_dict[service_key_old] + service_key_new = new_store.build_key(runstate.SERVICE_STATE, service) + new_store.save([(service_key_new, old_service_data)]) def copy_jobs(old_store, new_store, job_names): for job in job_names: job_key_old = old_store.build_key(runstate.JOB_STATE, job) - old_job_data = old_store.restore([job_key_old])[job_key_old] - job_state_key = new_store.build_key(runstate.JOB_STATE, job) + old_job_dict = old_store.restore([job_key_old]) + if old_job_dict: + old_job_data = old_job_dict[job_key_old] + job_state_key = new_store.build_key(runstate.JOB_STATE, job) - run_ids = [] - for job_run in old_job_data['runs']: - run_ids.append(job_run.run_num) - job_run_key = new_store.build_key(runstate.JOB_RUN_STATE, - job + ('.%s' % job_run.run_num)) - new_store.save([(job_run_key, job_run)]) + run_ids = [] + for job_run in old_job_data['runs']: + run_ids.append(job_run.run_num) + job_run_key = new_store.build_key(runstate.JOB_RUN_STATE, + job + ('.%s' % job_run.run_num)) + new_store.save([(job_run_key, job_run)]) - run_ids = sorted(run_ids, reverse=True) - job_state_data = {'enabled': old_job_data['enabled'], 'run_ids': run_ids} - new_store.save([(job_state_key, job_state_data)]) + run_ids = sorted(run_ids, reverse=True) + job_state_data = {'enabled': old_job_data['enabled'], 'run_ids': run_ids} + new_store.save([(job_state_key, job_state_data)]) def main(): + print('Parsing options...') (options, working_dir, new_fname) = parse_options() os.chdir(working_dir) + print('Parsing configuration file...') config = parse_config(options.conf_dir) state_info = config.get_master().state_persistence + print('Setting up the old state storing object...') old_store = get_old_state_store(state_info) + print('Setting up the new state storing object...') new_state_info = compile_new_info(options, state_info, new_fname) new_store = ParallelStore(new_state_info) + print('Copying metadata...') copy_metadata(old_store, new_store) + print('Copying service data...') copy_services(old_store, new_store, config.get_services().keys()) + print('Converting job data...') copy_jobs(old_store, new_store, config.get_jobs().keys()) + print('...done.') if __name__ == "__main__": main() diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 0cfd913f9..1b47eb80d 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -25,10 +25,10 @@ def _increment_counter(self): """A simple method to make sure that we don't indefinitely increase the id assigned to StoreRequests. """ - return self.id+1 if not self.id == MAX_MSG_ID else 0 + return self.id_counter+1 if not self.id_counter == MAX_MSG_ID else 0 def build(self, req_type, data_type, data): - new_request = StoreRequest(self.id_counter, req_type, data, self.serializer) + new_request = StoreRequest(self.id_counter, req_type, data_type, data, self.serializer) self.id_counter = self._increment_counter() return new_request diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index 0057f62fc..891d0d8b3 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -1,6 +1,7 @@ import itertools import operator import logging +import os from twisted.internet import reactor from tron.serialize.runstate.tronstore.process import StoreProcessProtocol @@ -51,15 +52,35 @@ def start_process(self): The command line arguments given to spawnProcess are in a HARDCODED ORDER that MUST match the order that tronstore parses them. """ - reactor.spawnProcess(self.process, "serialize/runstate/tronstore/tronstore", - ["tronstore", + path = os.path.dirname(msg_enums.__file__) + "/tronstore" + + pre_args = ["tronstore", self.config.name, self.config.transport_method, self.config.store_type, self.config.connection_details, self.config.db_store_method] + post_args = [] + for arg in pre_args: + post_args.append(arg if arg else 'None') + + # We need to make sure that the PYTHONPATH environment variable ISN'T + # relative! The working dir will be fine, the other environment + # variables either don't matter/are fine, but PYTHONPATH HAS to be + # set properly to avoid import errors! + # We can't use an absolute path conversion on ., as Tron changes the + # working directory to a specified (or default) parameter on startup. + environment = os.environ + real_pypath = msg_enums.__file__.split('/tron/serialize/runstate/tronstore/')[0] + environment['PYTHONPATH'] = real_pypath + + reactor.spawnProcess(self.process, + path, + args=post_args, + env=environment, + childFDs={0: "w", 1: "r", 2: 2} ) - reactor.run() + # reactor.run() def build_key(self, type, iden): return ParallelKey(type, iden) diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index f9d02acd7..4aaec5adf 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -70,7 +70,7 @@ def processExited(self, reason): raise TronStoreError(reason.getErrorMessage()) def processEnded(self, reason): - """Called by twisted whenever the process ends. Cleans up.""" + """Called by twisted whenever the process ends. Cleans up if needed.""" if not self.is_shutdown: self.transport.loseConnection() reactor.stop() diff --git a/tron/serialize/runstate/tronstore/tronstore b/tron/serialize/runstate/tronstore/tronstore index 54769d1a4..459a90ab2 100755 --- a/tron/serialize/runstate/tronstore/tronstore +++ b/tron/serialize/runstate/tronstore/tronstore @@ -22,14 +22,15 @@ from threading import Thread from Queue import Queue, Empty from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory -import tron.serialize.runstate.tronstore.store -import tron.serialize.runstate.tronstore.msg_enums -from tron.serialize.tronstore.chunking import StoreChunkHandler +from tron.serialize.runstate.tronstore import store +from tron.serialize.runstate.tronstore import msg_enums +from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler is_shutdown = False -def shutdown_handler(): +def shutdown_handler(signum, frame): """Signal the process to shut down, finishing any requests it already had.""" + global is_shutdown is_shutdown = True def parse_args(): @@ -53,6 +54,7 @@ def enqueue_input(stdin, queue): stdin.close() except Exception, e: # something happened, finish up and exit stdin.close() + global is_shutdown is_shutdown = True def start_stdin_thread(): @@ -95,6 +97,7 @@ def main(): chunk_handler = StoreChunkHandler() stdin_queue = start_stdin_thread() + global is_shutdown signal.signal(signal.SIGINT, shutdown_handler) while True: @@ -103,7 +106,7 @@ def main(): time.sleep(0.5) if stdin_queue.empty(): store_class.cleanup() - break + return requests = chunk_handler.handle(get_all_from_queue(stdin_queue)) requests = map(request_factory.rebuild, requests) From 9c1f0ed3ec365c743bed0ca279c74a5d8ac29139 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 17 Jul 2013 09:35:15 -0700 Subject: [PATCH 09/48] large refactor to not use twisted for tronstore --- .../runstate/tronstore/parallelstore_test.py | 49 ++- .../runstate/tronstore/process_test.py | 308 ++++++++++++------ .../migrate_state_from_0.6.1_to_0.6.2.py | 22 +- tron/serialize/runstate/tronstore/chunking.py | 2 +- tron/serialize/runstate/tronstore/messages.py | 32 +- .../serialize/runstate/tronstore/msg_enums.py | 4 +- .../runstate/tronstore/parallelstore.py | 59 +--- tron/serialize/runstate/tronstore/process.py | 172 +++++----- .../serialize/runstate/tronstore/transport.py | 4 +- tron/serialize/runstate/tronstore/tronstore | 121 ------- .../serialize/runstate/tronstore/tronstore.py | 202 ++++++++++++ 11 files changed, 567 insertions(+), 408 deletions(-) delete mode 100755 tron/serialize/runstate/tronstore/tronstore create mode 100644 tron/serialize/runstate/tronstore/tronstore.py diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py index 37818c10f..e4cdeb4f7 100644 --- a/tests/serialize/runstate/tronstore/parallelstore_test.py +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -18,28 +18,25 @@ def setup_store(self): db_store_method=None, buffer_size=1 ) - with contextlib.nested( - mock.patch('twisted.internet.reactor.spawnProcess', autospec=True), - mock.patch('twisted.internet.reactor.run', autospec=True), - mock.patch('tron.serialize.runstate.tronstore.parallelstore.StoreProcessProtocol', autospec=True) - ) as (self.spawn_patch, self.run_patch, self.process_patch): + with mock.patch('tron.serialize.runstate.tronstore.parallelstore.StoreProcessProtocol', autospec=True) \ + as (self.process_patch): self.store = ParallelStore(self.config) yield def test__init__(self): - self.process_patch.assert_called_once_with(self.store.response_factory) + self.process_patch.assert_called_once_with(self.store.path, self.config, self.store.response_factory) - def test_start_process(self): - self.spawn_patch.assert_called_once_with( - self.store.process, - "serialize/runstate/tronstore/tronstore", - ["tronstore", - self.config.name, - self.config.transport_method, - self.config.store_type, - self.config.connection_details, - self.config.db_store_method]) - self.run_patch.assert_called_once_with() + # def test_start_process(self): + # self.spawn_patch.assert_called_once_with( + # self.store.process, + # "serialize/runstate/tronstore/tronstore", + # ["tronstore", + # self.config.name, + # self.config.transport_method, + # self.config.store_type, + # self.config.connection_details, + # self.config.db_store_method]) + # self.run_patch.assert_called_once_with() def test_build_key(self): key_type = runstate.JOB_STATE @@ -61,7 +58,7 @@ def test_save(self): def test_restore_single_success(self): key = self.store.build_key(runstate.JOB_STATE, 'zeus_ult') - fake_response = mock.Mock(data=10, successful=True) + fake_response = mock.Mock(data=10, success=True) with contextlib.nested( mock.patch.object(self.store.request_factory, 'build'), mock.patch.object(self.store.process, 'send_request_get_response', return_value=fake_response), @@ -72,7 +69,7 @@ def test_restore_single_success(self): def test_restore_single_failure(self): key = self.store.build_key(runstate.JOB_STATE, 'rip_ryan_davis') - fake_response = mock.Mock(data=777, successful=False) + fake_response = mock.Mock(data=777, success=False) with contextlib.nested( mock.patch.object(self.store.request_factory, 'build'), mock.patch.object(self.store.process, 'send_request_get_response', return_value=fake_response), @@ -98,17 +95,15 @@ def test_cleanup(self): def test_load_config(self): new_config = mock.Mock() + config_req = mock.Mock() with contextlib.nested( mock.patch.object(self.store.request_factory, 'update_method'), mock.patch.object(self.store.response_factory, 'update_method'), - mock.patch.object(self.store.process, 'shutdown'), - mock.patch.object(self.store, 'start_process'), - mock.patch('tron.serialize.runstate.tronstore.process.StoreProcessProtocol', autospec=True) - ) as (request_patch, response_patch, shutdown_patch, start_patch, process_patch): + mock.patch.object(self.store.process, 'update_config'), + mock.patch.object(self.store.request_factory, 'build', return_value=config_req) + ) as (request_patch, response_patch, update_patch, build_patch): self.store.load_config(new_config) + build_patch.assert_called_once_with(msg_enums.REQUEST_CONFIG, '', new_config) request_patch.assert_called_once_with(new_config.transport_method) response_patch.assert_called_once_with(new_config.transport_method) - shutdown_patch.assert_called_once_with() - self.process_patch.assert_any_call(self.store.response_factory) - start_patch.assert_called_once_with() - assert_equal(self.store.config, new_config) + update_patch.assert_called_once_with(new_config, config_req) diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index 64a71be1e..768a99f75 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -1,148 +1,250 @@ import contextlib import mock from testify import TestCase, run, assert_equal, assert_raises, setup_teardown +from tron.serialize.runstate.tronstore import tronstore from tron.serialize.runstate.tronstore.process import StoreProcessProtocol, TronStoreError -from tron.serialize.runstate.tronstore import chunking class StoreProcessProtocolTestCase(TestCase): @setup_teardown def setup_process(self): - with mock.patch('twisted.internet.reactor.stop', autospec=True) as self.stop_patch: + self.test_pipe_a = mock.Mock() + self.test_pipe_b = mock.Mock() + pipe_return = mock.Mock(return_value=(self.test_pipe_a, self.test_pipe_b)) + with contextlib.nested( + mock.patch('tron.serialize.runstate.tronstore.process.Process', + autospec=True), + mock.patch('tron.serialize.runstate.tronstore.process.Pipe', + new=pipe_return) + ) as (self.process_patch, self.pipe_setup_patch): + self.config = mock.Mock( + name='test_config', + transport_method='pickle', + store_type='shelve', + connection_details=None, + db_store_method=None, + buffer_size=1 + ) + self.path = 'gaben/at/valve/software/dot/com' self.factory = mock.Mock() - self.process = StoreProcessProtocol(self.factory) + self.process = StoreProcessProtocol(self.path, self.config, self.factory) yield def test__init__(self): assert_equal(self.process.response_factory, self.factory) - assert_equal(self.process.requests, {}) - assert_equal(self.process.responses, {}) - assert_equal(self.process.semaphores, {}) + assert_equal(self.process.config, self.config) + assert_equal(self.process.orphaned_responses, {}) + assert_equal(self.process.path, self.path) assert not self.process.is_shutdown - def test_outRecieved_chunked(self): - data = 'deadly_premonition_br' - self.process.outRecieved(data) - assert_equal(self.process.chunker.chunk, data) - - def test_outRecieved_full_no_monitor(self): - data = 'no_one_str_should_have_all_that_power' + chunking.CHUNK_SIGNING_STR - fake_id = 77 - fake_response = mock.Mock(id=fake_id, success=True) - self.factory.rebuild = mock.Mock(return_value=fake_response) - self.process.requests[fake_id] = fake_response - self.process.outRecieved(data) - assert_equal(self.process.chunker.chunk, '') - assert_equal(self.process.responses, {}) - assert_equal(self.process.requests, {}) - assert_equal(self.process.semaphores, {}) - - def test_outRecieved_full_with_monitor(self): - data = 'throwing_dark' + chunking.CHUNK_SIGNING_STR - fake_id = 77 - fake_response = mock.Mock(id=fake_id, success=True) - self.factory.rebuild = mock.Mock(return_value=fake_response) - fake_semaphore = mock.Mock() - self.process.semaphores[fake_id] = fake_semaphore - self.process.requests[fake_id] = fake_response - self.process.outRecieved(data) - assert_equal(self.process.chunker.chunk, '') - assert_equal(self.process.responses, {fake_id: fake_response}) - assert_equal(self.process.semaphores, {fake_id: fake_semaphore}) - assert_equal(self.process.requests, {}) - fake_semaphore.release.assert_called_once_with() - - def test_processExited_running(self): - self.process.is_shutdown = False - assert_raises(TronStoreError, self.process.processExited, mock.Mock(getErrorMessage=lambda: 'test')) - - def test_processExited_shutdown(self): - self.process.is_shutdown = True - self.process.processExited('its_a_website') + def test_start_process(self): + self.pipe_setup_patch.assert_called_once_with() + self.process_patch.assert_called_once_with(target=tronstore.main, args=(self.config, self.test_pipe_b)) + assert self.process_patch.daemon + self.process.process.start.assert_called_once_with() - def test_processEnded_running(self): - self.process.is_shutdown = False - with mock.patch.object(self.process, 'transport') as lose_patch: - self.process.processEnded('about_videogames') - lose_patch.loseConnection.assert_called_once_with() - self.stop_patch.assert_called_once_with() + def test_verify_is_alive_while_dead(self): + with contextlib.nested( + mock.patch.object(self.process.process, 'is_alive', return_value=False), + mock.patch.object(self.process, '_start_process'), + ) as (alive_patch, start_patch): + assert_raises(TronStoreError, self.process._verify_is_alive) + alive_patch.assert_called_with() + assert_equal(alive_patch.call_count, 2) + start_patch.assert_called_once_with() - def test_processEnded_shutdown(self): - self.process.is_shutdown = True - with mock.patch.object(self.process, 'transport') as lose_patch: - self.process.processEnded('this_aint_no_game') - assert not self.stop_patch.called - assert not lose_patch.loseConnection.called + def test_verify_is_alive_while_alive(self): + with contextlib.nested( + mock.patch.object(self.process.process, 'is_alive', return_value=True), + mock.patch.object(self.process, '_start_process'), + ) as (alive_patch, start_patch): + self.process._verify_is_alive() + alive_patch.assert_called_once_with() + assert not start_patch.called def test_send_request_running(self): self.process.is_shutdown = False fake_id = 77 test_request = mock.Mock(serialized='sunny_sausalito', id=fake_id) with contextlib.nested( - mock.patch.object(self.process, 'transport'), - mock.patch.object(self.process.chunker, 'sign') - ) as (trans_patch, sign_patch): + mock.patch.object(self.process, '_verify_is_alive'), + mock.patch.object(self.process.pipe, 'send_bytes') + ) as (verify_patch, pipe_patch): self.process.send_request(test_request) - assert_equal(self.process.requests[fake_id], test_request) - sign_patch.assert_called_once_with(test_request.serialized) - trans_patch.write.assert_called_once_with(self.process.chunker.sign(test_request.serialized)) + verify_patch.assert_called_once_with() + pipe_patch.assert_called_once_with(test_request.serialized) def test_send_request_shutdown(self): self.process.is_shutdown = True fake_id = 77 test_request = mock.Mock(serialized='whiskey_media', id=fake_id) with contextlib.nested( - mock.patch.object(self.process, 'transport'), - mock.patch.object(self.process.chunker, 'sign') - ) as (trans_patch, sign_patch): + mock.patch.object(self.process, '_verify_is_alive'), + mock.patch.object(self.process.pipe, 'send_bytes') + ) as (verify_patch, pipe_patch): self.process.send_request(test_request) - assert_equal(self.process.requests, {}) - assert not sign_patch.called - assert not trans_patch.write.called + assert not verify_patch.called + assert not pipe_patch.called - def test_send_request_get_response_running(self): + def test_send_request_get_response_running_with_response(self): self.process.is_shutdown = False fake_id = 77 test_request = mock.Mock(serialized='objection', id=fake_id) test_response = mock.Mock(id=fake_id, data='overruled', success=True) - self.process.responses[fake_id] = test_response with contextlib.nested( - mock.patch.object(self.process, 'transport'), - mock.patch.object(self.process.chunker, 'sign'), - mock.patch('tron.serialize.runstate.tronstore.process.Semaphore', autospec=True) - ) as (trans_patch, sign_patch, sema_patch): - assert_equal(self.process.send_request_get_response(test_request), test_response.data) - assert_equal(self.process.requests, {fake_id: test_request}) - assert_equal(self.process.semaphores, {}) - assert_equal(self.process.responses, {}) - sign_patch.assert_called_once_with(test_request.serialized) - trans_patch.write.assert_called_once_with(self.process.chunker.sign(test_request.serialized)) - sema_patch.assert_called_once_with(0) + mock.patch.object(self.process, '_verify_is_alive'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=test_response) + ) as (verify_patch, pipe_patch, poll_patch): + assert_equal(self.process.send_request_get_response(test_request), test_response) + verify_patch.assert_called_once_with() + pipe_patch.assert_called_once_with(test_request.serialized) + poll_patch.assert_called_once_with(fake_id, self.process.POLL_TIMEOUT) + + def test_send_request_get_response_running_no_response(self): + self.process.is_shutdown = False + fake_id = 77 + test_request = mock.Mock(serialized='maaaaaagiiiiccc', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process, '_verify_is_alive'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=None) + ) as (verify_patch, pipe_patch, poll_patch): + assert_equal(self.process.send_request_get_response(test_request), + self.process.response_factory.build(False, fake_id, '')) + verify_patch.assert_called_once_with() + pipe_patch.assert_called_once_with(test_request.serialized) + poll_patch.assert_called_once_with(fake_id, self.process.POLL_TIMEOUT) def test_send_request_get_response_shutdown(self): self.process.is_shutdown = True fake_id = 77 - test_request = mock.Mock(serialized='does_he_look_like_a', id=fake_id) - test_response = mock.Mock(id=fake_id, data='what', success=True) - self.process.responses[fake_id] = test_response + test_request = mock.Mock(serialized='i_wish_for_the_nile', id=fake_id) + test_response = mock.Mock(id=fake_id, data='no_way', success=True) with contextlib.nested( - mock.patch.object(self.process, 'transport'), - mock.patch.object(self.process.chunker, 'sign'), - mock.patch('tron.serialize.runstate.tronstore.process.Semaphore', autospec=True) - ) as (trans_patch, sign_patch, sema_patch): - assert_equal(self.process.send_request_get_response(test_request), None) - assert_equal(self.process.requests, {}) - assert_equal(self.process.semaphores, {}) - assert_equal(self.process.responses, {fake_id: test_response}) - assert not sign_patch.called - assert not trans_patch.write.called - assert not sema_patch.called - - def test_shutdown(self): + mock.patch.object(self.process, '_verify_is_alive'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=test_response) + ) as (verify_patch, pipe_patch, poll_patch): + assert_equal(self.process.send_request_get_response(test_request), + self.process.response_factory.build(False, fake_id, '')) + assert not verify_patch.called + assert not pipe_patch.called + assert not poll_patch.called + + def test_send_request_shutdown_not_shutdown(self): self.process.is_shutdown = False - with mock.patch.object(self.process, 'transport') as trans_patch: - self.process.shutdown() + fake_id = 77 + test_request = mock.Mock(serialized='ghost_truck', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process.process, 'is_alive', return_value=True), + mock.patch.object(self.process.pipe, 'close'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=mock.Mock()), + mock.patch.object(self.process.process, 'terminate') + ) as (alive_patch, close_patch, send_patch, poll_patch, terminate_patch): + self.process.send_request_shutdown(test_request) + alive_patch.assert_called_once_with() assert self.process.is_shutdown - trans_patch.signalProcess.assert_called_once_with('INT') - trans_patch.loseConnection.assert_called_once_with() - self.stop_patch.assert_called_once_with() + send_patch.assert_called_once_with(test_request.serialized) + poll_patch.assert_called_once_with(fake_id, self.process.SHUTDOWN_TIMEOUT) + close_patch.assert_called_once_with() + terminate_patch.assert_called_once_with() + + def test_send_request_shutdown_is_shutdown(self): + self.process.is_shutdown = True + fake_id = 77 + test_request = mock.Mock(serialized='thats_million_bucks', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process.process, 'is_alive', return_value=True), + mock.patch.object(self.process.pipe, 'close'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=mock.Mock()), + mock.patch.object(self.process.process, 'terminate') + ) as (alive_patch, close_patch, send_patch, poll_patch, terminate_patch): + self.process.send_request_shutdown(test_request) + assert not alive_patch.called # should have short circuited + close_patch.assert_called_once_with() + assert self.process.is_shutdown + assert not send_patch.called + assert not poll_patch.called + assert not terminate_patch.called + + def test_send_request_shutdown_not_shutdown_but_dead(self): + self.process.is_shutdown = False + fake_id = 77 + test_request = mock.Mock(serialized='thats_million_bucks', id=fake_id) + with contextlib.nested( + mock.patch.object(self.process.process, 'is_alive', return_value=False), + mock.patch.object(self.process.pipe, 'close'), + mock.patch.object(self.process.pipe, 'send_bytes'), + mock.patch.object(self.process, '_poll_for_response', return_value=mock.Mock()), + mock.patch.object(self.process.process, 'terminate') + ) as (alive_patch, close_patch, send_patch, poll_patch, terminate_patch): + self.process.send_request_shutdown(test_request) + alive_patch.assert_called_once_with() + close_patch.assert_called_once_with() + assert self.process.is_shutdown + assert not send_patch.called + assert not poll_patch.called + assert not terminate_patch.called + + def test_update_config(self): + request = mock.Mock() + new_config = mock.Mock() + with mock.patch.object(self.process, 'send_request') as send_patch: + self.process.update_config(new_config, request) + send_patch.assert_called_once_with(request) + assert_equal(self.process.config, new_config) + + def test_poll_for_response_has_response_makes_orphaned(self): + self.process.orphaned_responses = {} + fake_id = 77 + fake_timeout = 0.05 + fake_response_serial = ['first'] + fake_response_matching = mock.Mock(serialized=fake_response_serial, id=fake_id) + fake_id_other = 96943 + fake_response_other = mock.Mock(serialized='oliver', id=fake_id_other) + + def recv_change_response(): + ret = fake_response_serial[0] + fake_response_serial[0] = 'second' + return ret + + def get_fake_response(fake_response_serial): + if fake_response_serial == 'first': + return fake_response_other + else: + return fake_response_matching + + with contextlib.nested( + mock.patch.object(self.process.pipe, 'poll', return_value=True), + mock.patch.object(self.process.pipe, 'recv_bytes', side_effect=recv_change_response), + mock.patch.object(self.process.response_factory, 'rebuild', side_effect=get_fake_response) + ) as (poll_patch, recv_patch, rebuild_patch): + assert_equal(self.process._poll_for_response(fake_id, fake_timeout), fake_response_matching) + assert_equal(self.process.orphaned_responses, {fake_id_other: fake_response_other}) + poll_patch.assert_called_with(fake_timeout) + assert_equal(poll_patch.call_count, 2) + recv_patch.assert_called_with() + assert_equal(recv_patch.call_count, 2) + rebuild_patch.assert_called_with('second') + assert_equal(rebuild_patch.call_count, 2) + + def test_poll_for_response_has_orphaned(self): + fake_id = 77 + fake_timeout = 0.05 + fake_response = mock.Mock(serialized='wherein_there_is_dotes', id=fake_id) + self.process.orphaned_responses = {fake_id: fake_response} + with contextlib.nested( + mock.patch.object(self.process.pipe, 'poll', return_value=True), + mock.patch.object(self.process.pipe, 'recv_bytes'), + mock.patch.object(self.process.response_factory, 'rebuild') + ) as (poll_patch, recv_patch, rebuild_patch): + assert_equal(self.process._poll_for_response(fake_id, fake_timeout), fake_response) + assert_equal(self.process.orphaned_responses, {}) + assert not poll_patch.called + assert not recv_patch.called + assert not rebuild_patch.called + + def test_poll_for_response_no_response(self): diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index f47323931..9869a640b 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -113,20 +113,20 @@ def get_old_state_store(state_info): def compile_new_info(options, state_info, new_file): new_state_info = copy.deepcopy(state_info) - new_state_info._replace(name=new_file) + new_state_info = new_state_info._replace(name=new_file) if options.store_method: - new_state_info._replace(store_method=options.store_method) + new_state_info = new_state_info._replace(store_method=options.store_method) if options.transport_method: - new_state_info._replace(transport_method=options.transport_method) + new_state_info = new_state_info._replace(transport_method=options.transport_method) if options.db_store_method: - new_state_info._replace(db_store_method=options.db_store_method) + new_state_info = new_state_info._replace(db_store_method=options.db_store_method) if options.new_connection_details \ and options.new_connection_details != state_info.connection_details: - new_state_info._replace(connection_details=options.new_connection_details) + new_state_info = new_state_info._replace(connection_details=options.new_connection_details) elif new_state_info.store_type in ('sql', 'mongo'): raise ConfigError('Must specify new connection_details using -c to use %s' % new_state_info.store_type) @@ -140,17 +140,17 @@ def copy_metadata(old_store, new_store): old_metadata = old_metadata_dict[meta_key_old] meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) - assert old_metadata == new_store.restore([meta_key_new]) + assert old_metadata == new_store.restore([meta_key_new])[meta_key_new] def copy_services(old_store, new_store, service_names): for service in service_names: service_key_old = old_store.build_key(runstate.SERVICE_STATE, service) - import ipdb; ipdb.set_trace() old_service_dict = old_store.restore([service_key_old]) if old_service_dict: old_service_data = old_service_dict[service_key_old] service_key_new = new_store.build_key(runstate.SERVICE_STATE, service) new_store.save([(service_key_new, old_service_data)]) + assert old_service_data == new_store.restore([service_key_new])[service_key_new] def copy_jobs(old_store, new_store, job_names): for job in job_names: @@ -162,14 +162,16 @@ def copy_jobs(old_store, new_store, job_names): run_ids = [] for job_run in old_job_data['runs']: - run_ids.append(job_run.run_num) + run_ids.append(job_run['run_num']) job_run_key = new_store.build_key(runstate.JOB_RUN_STATE, - job + ('.%s' % job_run.run_num)) + job + ('.%s' % job_run['run_num'])) new_store.save([(job_run_key, job_run)]) + assert job_run == new_store.restore([job_run_key])[job_run_key] run_ids = sorted(run_ids, reverse=True) job_state_data = {'enabled': old_job_data['enabled'], 'run_ids': run_ids} new_store.save([(job_state_key, job_state_data)]) + assert job_state_data == new_store.restore([job_state_key])[job_state_key] def main(): @@ -192,6 +194,8 @@ def main(): print('Converting job data...') copy_jobs(old_store, new_store, config.get_jobs().keys()) print('...done.') + old_store.cleanup() + new_store.cleanup() if __name__ == "__main__": main() diff --git a/tron/serialize/runstate/tronstore/chunking.py b/tron/serialize/runstate/tronstore/chunking.py index 626756d61..0abe78c2d 100644 --- a/tron/serialize/runstate/tronstore/chunking.py +++ b/tron/serialize/runstate/tronstore/chunking.py @@ -5,7 +5,7 @@ class StoreChunkHandler(object): messaging. Works by one end using the sign() function to sign the serialized string and then using handle() on the opposite end of the wire. - This is used by tronstore, as the twisted stdin/out handler chunks.""" + This is used by tronstore, as the pipes can get muddled.""" def __init__(self): self.chunk = '' diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 1b47eb80d..c8524331c 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -11,11 +11,7 @@ class StoreRequestFactory(object): - """A factory to generate requests that need to be converted to serialized - strings and back. All the factory itself does is keep track of what - serialization method was set by the configuration, and then constructs - specific StoreRequest objects using that method. - """ + """A factory to generate requests by giving each a unique id.""" def __init__(self, method): self.serializer = transport_class_map[method] @@ -79,12 +75,12 @@ class StoreRequest(object): """ def __init__(self, req_id, req_type, data_type, data, method): - self.id = req_id - self.req_type = req_type - self.data = data - self.data_type = data_type - self.method = method - self.serialized = self.get_serialized() + self.id = req_id + self.req_type = req_type + self.data = data + self.data_type = data_type + self.method = method + self.serialized = self.get_serialized() @classmethod def from_message(cls, msg_data, method): @@ -97,7 +93,11 @@ def update_method(self, new_method): self.serialized = self.get_serialized() def get_serialized(self): - return self.method.serialize((self.id, self.req_type, self.data_type, self.data)) + return self.method.serialize(( + self.id, + self.req_type, + self.data_type, + self.data)) class StoreResponse(object): @@ -109,10 +109,10 @@ class StoreResponse(object): """ def __init__(self, req_id, success, data, method): - self.id = req_id - self.success = success - self.data = data - self.method = method + self.id = req_id + self.success = success + self.data = data + self.method = method self.serialized = self.get_serialized() @classmethod diff --git a/tron/serialize/runstate/tronstore/msg_enums.py b/tron/serialize/runstate/tronstore/msg_enums.py index 30c2f29ac..b7de4cee6 100644 --- a/tron/serialize/runstate/tronstore/msg_enums.py +++ b/tron/serialize/runstate/tronstore/msg_enums.py @@ -1,4 +1,4 @@ REQUEST_SAVE = 10 REQUEST_RESTORE = 11 -# REQUEST_CONFIG = 12 # Probably not needed... -# REQUEST_SHUTDOWN = 13 # Not needed, using SIGINT +REQUEST_CONFIG = 12 +REQUEST_SHUTDOWN = 13 diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index 891d0d8b3..cdae280d5 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -3,7 +3,7 @@ import logging import os -from twisted.internet import reactor +# from twisted.internet import reactor from tron.serialize.runstate.tronstore.process import StoreProcessProtocol from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory from tron.serialize.runstate.tronstore import msg_enums @@ -40,47 +40,10 @@ class ParallelStore(object): to tronstore based on requests given by the MCP.""" def __init__(self, config): - self.config = config self.request_factory = StoreRequestFactory(config.transport_method) self.response_factory = StoreResponseFactory(config.transport_method) - self.process = StoreProcessProtocol(self.response_factory) - self.start_process() - - def start_process(self): - """Use twisted to spawn the tronstore process. - - The command line arguments given to spawnProcess are in a - HARDCODED ORDER that MUST match the order that tronstore parses them. - """ - path = os.path.dirname(msg_enums.__file__) + "/tronstore" - - pre_args = ["tronstore", - self.config.name, - self.config.transport_method, - self.config.store_type, - self.config.connection_details, - self.config.db_store_method] - post_args = [] - for arg in pre_args: - post_args.append(arg if arg else 'None') - - # We need to make sure that the PYTHONPATH environment variable ISN'T - # relative! The working dir will be fine, the other environment - # variables either don't matter/are fine, but PYTHONPATH HAS to be - # set properly to avoid import errors! - # We can't use an absolute path conversion on ., as Tron changes the - # working directory to a specified (or default) parameter on startup. - environment = os.environ - real_pypath = msg_enums.__file__.split('/tron/serialize/runstate/tronstore/')[0] - environment['PYTHONPATH'] = real_pypath - - reactor.spawnProcess(self.process, - path, - args=post_args, - env=environment, - childFDs={0: "w", 1: "r", 2: 2} - ) - # reactor.run() + self.path = os.path.dirname(msg_enums.__file__) + "/tronstore" + self.process = StoreProcessProtocol(self.path, config, self.response_factory) def build_key(self, type, iden): return ParallelKey(type, iden) @@ -93,30 +56,24 @@ def save(self, key_value_pairs): def restore_single(self, key): request = self.request_factory.build(msg_enums.REQUEST_RESTORE, key.type, key.key) response = self.process.send_request_get_response(request) - return response.data if response.successful else None + return response.data if response.success else None def restore(self, keys): items = itertools.izip(keys, (self.restore_single(key) for key in keys)) return dict(itertools.ifilter(operator.itemgetter(1), items)) def cleanup(self): - self.process.shutdown() + shutdown_req = self.request_factory.build(msg_enums.REQUEST_SHUTDOWN, '', '') + self.process.send_request_shutdown(shutdown_req) shutdown = cleanup - # This method may not be needed. From looking at the StateChangeWatcher - # implementation, it looks like it makes a completely new instance of a - # PersistentStateManager whenever the config is updated, which removes - # the need for changing config related things here (since a new instance - # of this class will be created anyway). def load_config(self, new_config): """Reconfigure the storing mechanism to use a new configuration by shutting down and restarting tronstore.""" - self.config = new_config + config_req = self.request_factory.build(msg_enums.REQUEST_CONFIG, '', new_config) + self.process.update_config(new_config, config_req) self.request_factory.update_method(new_config.transport_method) - self.process.shutdown() self.response_factory.update_method(new_config.transport_method) - self.process = StoreProcessProtocol(self.response_factory) - self.start_process() def __repr__(self): return "ParallelStore" diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index 4aaec5adf..dcb2a99b0 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -1,10 +1,10 @@ import time import logging -from threading import Semaphore +# import os +from multiprocessing import Process, Pipe -from twisted.internet.protocol import ProcessProtocol -from twisted.internet import reactor -from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler +from tron.serialize.runstate.tronstore import tronstore +# from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler log = logging.getLogger(__name__) @@ -15,7 +15,7 @@ def __init__(self, code): def __str__(self): return repr(self.code) -class StoreProcessProtocol(ProcessProtocol): +class StoreProcessProtocol(object): """The class that actually communicates with tronstore. This is a subclass of the twisted ProcessProtocol class, which has a set of internals that can communicate with a child proccess via stdin/stdout via interrupts. @@ -25,94 +25,114 @@ class StoreProcessProtocol(ProcessProtocol): while requests have an enumerator (see msg_enums.py) to identify the type of request. """ + # This timeout MUST be longer than the one in tronstore! + SHUTDOWN_TIMEOUT = 100.0 + POLL_TIMEOUT = 10.0 - SHUTDOWN_TIMEOUT = 5.0 - SHUTDOWN_SLEEP = 0.5 - - def __init__(self, response_factory): + def __init__(self, path, config, response_factory): + self.config = config self.response_factory = response_factory - self.chunker = StoreChunkHandler() - self.requests = {} - self.responses = {} - self.semaphores = {} # semaphores used for synchronization + self.orphaned_responses = {} + self.path = path self.is_shutdown = False + self._start_process() - def outRecieved(self, data): - """Called via interrupt whenever twisted sees something written by the - process into stdout, where data is whatever the process wrote. - Since the only thing written to stdout are serialized responses from - tronstore, this method deals with matching the response to the - appropriate request if the request needed it. - - As some requests actually require a response (see: restore requests), - this method also wakes up the main trond thread if it was blocking on a - response from tronstore. - """ - responses = self.chunker.handle(data) - for response_str in responses: - response = self.response_factory.rebuild(response_str) - # Requests that don't actually require a response don't put - # themselves inside of the requests dict. - if response.id in self.requests: - if not response.success: - log.warn("tronstore request #%d failed. Request type was %d." % (response.id, self.requests[response.id].req_type)) - if response.id in self.semaphores: - self.responses[response.id] = response - self.semaphores[response.id].release() - del self.requests[response.id] - - def processExited(self, reason): - """Called by twisted whenever the process exits. - If the process didn't exit cleanly (we didn't shut it down), - then we need to raise an exception. + def _start_process(self): + """Spawn the tronstore process. The arguments given to tronstore must + match the signature for tronstore.main. """ - if not self.is_shutdown: - raise TronStoreError(reason.getErrorMessage()) + self.pipe, child_pipe = Pipe() + store_args = (self.config, child_pipe) - def processEnded(self, reason): - """Called by twisted whenever the process ends. Cleans up if needed.""" - if not self.is_shutdown: - self.transport.loseConnection() - reactor.stop() + self.process = Process(target=tronstore.main, args=store_args) + self.process.daemon = True + self.process.start() + + def _verify_is_alive(self): + """A check to verify that tronstore is alive. Attempts to restart + tronstore if it finds that it exited for some reason.""" + if not self.process.is_alive(): + code = self.process.exitcode + log.warn("tronstore exited prematurely with status code %d. Attempting to restart." % code) + self._start_process() + if not self.process.is_alive(): + raise TronStoreError("tronstore crashed with status code %d and failed to restart" % code) def send_request(self, request): - """Send a request to tronstore and immediately return without + """Send a StoreRequest to tronstore and immediately return without waiting for tronstore's response. """ if self.is_shutdown: return - self.requests[request.id] = request - self.transport.write(self.chunker.sign(request.serialized)) + self._verify_is_alive() + + self.pipe.send_bytes(request.serialized) + # self.transport.write(self.chunker.sign(request.serialized)) + + def _poll_for_response(self, id, timeout): + """Polls for a response to the request with identifier id. Throws + any responses that it isn't looking for into a dict, and tries to + retrieve a matching response from this dict before pulling new + responses. + + If Tron is extended into a synchronous program, simply just add a + lock around this function ( with mutex.lock(): ) and everything'll + be fine. + """ + if id in self.orphaned_responses: + response = self.orphaned_responses[id] + del self.orphaned_responses[id] + return response + + while self.pipe.poll(timeout): + response = self.response_factory.rebuild(self.pipe.recv_bytes()) + if response.id == id: + return response + else: + self.orphaned_responses[response.id] = response + return None def send_request_get_response(self, request): - """Send a request to tronstore, and block until tronstore responds - with the appropriate data. If the request was successful, we return - whatever data tronstore sent us, otherwise, None is returned. + """Send a StoreRequest to tronstore, and block until tronstore responds + with the appropriate data. The StoreResponse is returned as is, with no + modifications. Blocks for POLL_TIMEOUT seconds until returning None. """ + if self.is_shutdown: - return None - self.requests[request.id] = request - self.semaphores[request.id] = Semaphore(0) - self.transport.write(self.chunker.sign(request.serialized)) - self.semaphores[request.id].acquire() - del self.semaphores[request.id] - response = self.responses[request.id] - del self.responses[request.id] - return response.data if response.success else None - - def shutdown(self): + return self.response_factory.build(False, request.id, '') + self._verify_is_alive() + + self.pipe.send_bytes(request.serialized) + response = self._poll_for_response(request.id, self.POLL_TIMEOUT) + if not response: + log.warn("tronstore took longer than %d seconds to respond to a request, and it was dropped." % self.POLL_TIMEOUT) + return self.response_factory.build(False, request.id, '') + else: + return response + + def send_request_shutdown(self, request): """Shut down the process protocol. Waits for SHUTDOWN_TIMEOUT seconds - for all pending requests to get responses from tronstore, after which - it cuts the connection. It checks if all requests have been completed - every SHUTDOWN_SLEEP seconds. + for tronstore to send a response, after which it kills both pipes + and the process itself. - Calling this prevents ANY further requests being made to tronstore. + Calling this prevents ANY further requests being made to tronstore, as + the process will be terminated. """ + if self.is_shutdown or not self.process.is_alive(): + self.pipe.close() + self.is_shutdown = True + return self.is_shutdown = True - time_waited = 0 - while (not len(self.requests.items()) == 0) and time_waited < self.SHUTDOWN_TIMEOUT: - time.sleep(self.SHUTDOWN_SLEEP) # wait for all pending requests to finish - time_waited += self.SHUTDOWN_SLEEP - self.transport.signalProcess("INT") - self.transport.loseConnection() - reactor.stop() + + self.pipe.send_bytes(request.serialized) + response = self._poll_for_response(request.id, self.SHUTDOWN_TIMEOUT) + + if not response or not response.success: + log.error("tronstore failed to shut down successfully.") + + self.pipe.close() + self.process.terminate() + + def update_config(self, new_config, config_request): + self.send_request(config_request) + self.config = new_config diff --git a/tron/serialize/runstate/tronstore/transport.py b/tron/serialize/runstate/tronstore/transport.py index b8f174ec6..e696dcbdf 100644 --- a/tron/serialize/runstate/tronstore/transport.py +++ b/tron/serialize/runstate/tronstore/transport.py @@ -33,7 +33,7 @@ def __str__(self): class JSONTransport(object): @classmethod def serialize(cls, data): - return json.dumps(data) + return json.dumps(data, tuple_as_array=False) @classmethod def deserialize(cls, data_str): @@ -55,7 +55,7 @@ class MsgPackTransport(object): def serialize(cls, data): if no_msgpack: raise TransportModuleError('MessagePack not installed.') - return msgpack.packb(data, use_list=False) + return msgpack.packb(data) @classmethod def deserialize(cls, data_str): diff --git a/tron/serialize/runstate/tronstore/tronstore b/tron/serialize/runstate/tronstore/tronstore deleted file mode 100755 index 459a90ab2..000000000 --- a/tron/serialize/runstate/tronstore/tronstore +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -"""This process is spawned by trond in order to offload state save/load -operations such that trond can focus on the more important things without -blocking for large chunks of time. It takes command line arguments (the order -of which is specifically hardcoded) in order to configure itself and use -the correct methods for state saving and message transport with trond. - -Messages are sent via stdin/out, as the twisted framework that provides -management of child processes uses it for communication. Because the core -Python stdin reader is blocking, a separate thread is spawned in tronstore -to buffer all messages that come via stdin. - -The process gracefully shuts down whenever it recieves a SIGINT by first -processing all requests it has already read from stdin. trond capitalizes on -this behavior by propagating any SIGINT signals it receives to tronstore. -""" -import sys -import os -import signal -import time -from threading import Thread -from Queue import Queue, Empty - -from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory -from tron.serialize.runstate.tronstore import store -from tron.serialize.runstate.tronstore import msg_enums -from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler - -is_shutdown = False - -def shutdown_handler(signum, frame): - """Signal the process to shut down, finishing any requests it already had.""" - global is_shutdown - is_shutdown = True - -def parse_args(): - """Parse the command line arguments. THIS MUST MATCH THE ORDER GIVEN BY - parallelstore.py when it initializes the tronstore process. - """ - name = sys.argv[1] - transport_method = sys.argv[2] - store_type = sys.argv[3] - connection_details = sys.argv[4] - db_store_method = sys.argv[5] - - return (transport_method, store.build_store(name, store_type, connection_details, db_store_method)) - -def enqueue_input(stdin, queue): - """Enqueue anything read by stdin into a queue. This is run in a separate - thread such that requests can be processed without blocking.""" - try: - for line in stdin: - queue.put(line) - stdin.close() - except Exception, e: # something happened, finish up and exit - stdin.close() - global is_shutdown - is_shutdown = True - -def start_stdin_thread(): - """Starts the thread that will enqueue all stdin data into the stdin_queue.""" - stdin_queue = Queue() - stdin_thread = Thread(target=enqueue_input, args=(sys.stdin, stdin_queue)) - stdin_thread.daemon = True - stdin_thread.start() - return stdin_queue - -def get_all_from_queue(queue): - """Gets all of the requests from the stdin_queue, returning one long string.""" - tmp_str = '' - while not queue.empty(): - try: - tmp_str += queue.get_nowait() - except Empty: - break - return tmp_str - -def handle_request(request, store_class): - """Handle a request by acting on store_class with the appropriate action.""" - if request.req_type == msg_enums.REQUEST_SAVE: - success = store_class.save(request.data[0], request.data[1], request.data_type) - return (success, request.id, '') - elif request.req_type == msg_enums.REQUEST_RESTORE: - success, data = store_class.restore(request.data, request.data_type) - return (success, request.id, data) - else: - return (False, request.id, '') - -def main(): - """The main run loop for tronstore. This loop sets up everything - based on the command line arguments tronstore got, and then simply - waits for requests to handle from stdin, and writes responses to - stdout. The process will run until an error or SIGINT occurs.""" - (transport_method, store_class) = parse_args() - request_factory = StoreRequestFactory(transport_method) - response_factory = StoreResponseFactory(transport_method) - chunk_handler = StoreChunkHandler() - stdin_queue = start_stdin_thread() - - global is_shutdown - signal.signal(signal.SIGINT, shutdown_handler) - - while True: - if is_shutdown: - if stdin_queue.empty(): - time.sleep(0.5) - if stdin_queue.empty(): - store_class.cleanup() - return - - requests = chunk_handler.handle(get_all_from_queue(stdin_queue)) - requests = map(request_factory.rebuild, requests) - for request in requests: - request = request_factory.rebuild(request) - response = handle_request(request, store_class) - response = response_factory.build(response[0], response[1], response[2]) - sys.stdout.write(chunk_handler.sign(response.serialized)) - - -if __name__ == '__main__': - main() diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py new file mode 100644 index 000000000..c3a1ca21c --- /dev/null +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +"""This process is spawned by trond in order to offload state save/load +operations such that trond can focus on the more important things without +blocking for large chunks of time. It takes arguments in the main method +passed by python's multiprocessing module in order to configure itself and use +the correct methods for state saving and message transport with trond. + +Messages are sent via Pipes (also part of python's multiprocessing module). +This allows for easy polling and no need to handle chunking of messages. + +The process intercepts the two shutdown signals (SIGINT and SIGTERM) in order +to prevent the process from exiting early when trond wants to do some final +shutdown things (realistically, it should be handling all shutdown operations +as this is a child process.) +""" +import time +import signal +from threading import Thread, Lock +from Queue import Queue, Empty + +from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory +from tron.serialize.runstate.tronstore import store +from tron.serialize.runstate.tronstore import msg_enums + +def shutdown_handler(signum, frame): + """This is just here to stop tronstore from exiting early. The process + will be terminated from the main tron daemon when all requests have been + finished. This is needed because Python propogates signals to + spawned processes, and tronstore is going to get a TON of requests whenever + a SIGINT is sent to the main daemon, as it has to save everything before + it can shut down.""" + pass + + +def parse_config(config): + """Parse the configuration file and set up the store class.""" + name = config.name + transport_method = config.transport_method + store_type = config.store_type + connection_details = config.connection_details + db_store_method = config.db_store_method + + return (store.build_store(name, store_type, connection_details, db_store_method), transport_method) + +# def enqueue_input(stdin, queue): +# """Enqueue anything read by stdin into a queue. This is run in a separate +# thread such that requests can be processed without blocking.""" +# open('tronstore_is_running.test', 'w') +# try: +# for line in stdin: +# open('tronstore_got_a_line.test', 'w') +# f.write('I read a line!') +# f.close() +# stdin.close() +# except Exception, e: # something happened, finish up and exit +# open('tronstore_stdin_crashed.test', 'w') +# stdin.close() +# global is_shutdown +# is_shutdown = True + +# def start_stdin_thread(): +# """Starts the thread that will enqueue all stdin data into the stdin_queue.""" +# stdin_queue = Queue() +# stdin_thread = Thread(target=enqueue_input, args=(sys.stdin, stdin_queue)) +# stdin_thread.daemon = True +# stdin_thread.start() +# return stdin_queue + +def get_all_from_pipe(pipe): + """Gets all of the requests from the pipe, returning an array of serialized + requests (they still need to be decoded). + """ + requests = [] + while pipe.poll(): + requests.append(pipe.recv_bytes()) + return requests + +def handle_request(request, store_class, pipe, factory, save_lock, restore_lock): + """Handle a request by acting on store_class with the appropriate action. + + This is run in a separate thread. As such, there's two mutexes here- one + for the save requests, and one for the restore requests.""" + + if request.req_type == msg_enums.REQUEST_SAVE: + with save_lock: + success = store_class.save(request.data[0], request.data[1], request.data_type) + pipe.send_bytes(factory.build(success, request.id, '').serialized) + + elif request.req_type == msg_enums.REQUEST_RESTORE: + with restore_lock: + success, data = store_class.restore(request.data, request.data_type) + pipe.send_bytes(factory.build(success, request.id, data).serialized) + + else: + pipe.send_bytes(factory.build(False, request.id, '').serialized) + +def thread_starter(queue, running_threads): + """A method to start threads that have been queued up in queue. Also takes + a reference to a list (running_threads) that this function will store any + threads it has started in, so the main method knows if there's still + currently executing requests. + + Keep in mind because running_threads is a reference to a single instance + of a list object, it CANNOT be reassigned to another instance in order to + allow the main thread to know what's running. As such, all operations on + running_threads must be method calls to modify the list instance given + to this thread.""" + global is_shutdown + POOL_SIZE = 35 + + pool_counter = POOL_SIZE + + def _remove_finished_threads(running_threads): + # A small helper function to clean out the running_threads array. + counter = 0 + for thread in running_threads: + if not thread.is_alive(): + running_threads.remove(thread) + counter += 1 + return counter + + while not is_shutdown or not queue.empty(): + pool_counter += _remove_finished_threads(running_threads) + + if pool_counter <= 0: + time.sleep(0.5) + continue + + try: + thread = queue.get(timeout=0.5) + thread.start() + pool_counter -= 1 + running_threads.append(thread) + except Empty: + continue + + while len(running_threads) != 0: + _remove_finished_threads(running_threads) + + +def main(config, pipe): + """The main run loop for tronstore. This loop sets up everything + based on the configuration tronstore got, and then simply + waits for requests to handle from pipe. It spawns threads for + save and restore requests, which will send responses back over + the pipe once completed.""" + global is_shutdown + # This timeout MUST BE SHORTER than the one in process.py! + # Seriously, if this is longer, everything will break! + SHUTDOWN_TIMEOUT = 3.0 + is_shutdown = False + + store_class, transport_method = parse_config(config) + + request_factory = StoreRequestFactory(transport_method) + response_factory = StoreResponseFactory(transport_method) + save_lock = Lock() + restore_lock = Lock() + + running_threads = [] + thread_queue = Queue() + thread_pool = Thread(target=thread_starter, args=(thread_queue, running_threads)) + thread_pool.start() + + signal.signal(signal.SIGINT, shutdown_handler) + signal.signal(signal.SIGTERM, shutdown_handler) + + while True: + timeout = SHUTDOWN_TIMEOUT if is_shutdown else None + if pipe.poll(timeout): + requests = get_all_from_pipe(pipe) + requests = map(request_factory.rebuild, requests) + for request in requests: + if request.req_type == msg_enums.REQUEST_SHUTDOWN: + is_shutdown = True + shutdown_req_id = request.id + + elif request.req_type == msg_enums.REQUEST_CONFIG: + while len(running_threads) != 0: + time.sleep(0.5) + store_class.cleanup() + store_class, transport_method = parse_config(request.data) + request_factory.update_method(transport_method) + response_factory.update_method(transport_method) + + else: + request_thread = Thread(target=handle_request, + args=( + request, + store_class, + pipe, + response_factory, + save_lock, + restore_lock)) + thread_queue.put(request_thread) + else: + # We have to wait for all requests to clean up first. + while len(running_threads) != 0: + time.sleep(0.5) + store_class.cleanup() + pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) + return From 74ff784d728d3c807cc7e0c5a335d02964255eff Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 18 Jul 2013 11:45:19 -0700 Subject: [PATCH 10/48] added unit tests for tronstore.py --- .../runstate/tronstore/process_test.py | 13 + .../runstate/tronstore/tronstore_test.py | 297 ++++++++++++++++++ .../serialize/runstate/tronstore/tronstore.py | 63 ++-- 3 files changed, 333 insertions(+), 40 deletions(-) create mode 100644 tests/serialize/runstate/tronstore/tronstore_test.py diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index 768a99f75..a305bcf92 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -248,3 +248,16 @@ def test_poll_for_response_has_orphaned(self): assert not rebuild_patch.called def test_poll_for_response_no_response(self): + fake_id = 77 + fake_timeout = 0.05 + self.process.orphaned_responses = {} + with contextlib.nested( + mock.patch.object(self.process.pipe, 'poll', return_value=False), + mock.patch.object(self.process.pipe, 'recv_bytes'), + mock.patch.object(self.process.response_factory, 'rebuild') + ) as (poll_patch, recv_patch, rebuild_patch): + assert_equal(self.process._poll_for_response(fake_id, fake_timeout), None) + assert_equal(self.process.orphaned_responses, {}) + poll_patch.assert_called_once_with(fake_timeout) + assert not recv_patch.called + assert not rebuild_patch.called diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py new file mode 100644 index 000000000..c05d1607d --- /dev/null +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -0,0 +1,297 @@ +import contextlib +import mock +import signal +from Queue import Queue +from testify import TestCase, run, assert_equal, assert_raises, setup_teardown, setup + +from tron.serialize.runstate.tronstore import tronstore, msg_enums + +class TronstoreMainTestCase(TestCase): + + @setup_teardown + def setup_main(self): + self.config = mock.Mock() + self.pipe = mock.Mock() + self.store_class = mock.Mock() + self.trans_method = mock.Mock() + self.mock_thread = mock.Mock() + self.request_factory = mock.Mock() + self.response_factory = mock.Mock() + self.lock = mock.Mock() + self.queue = mock.Mock() + + def poll_patch(timeout): + return False if timeout else True + self.pipe.poll = poll_patch + + def echo_single_request(request): + return request + self.request_factory.rebuild = echo_single_request + + def echo_requests(not_used): + return self.requests + + with contextlib.nested( + mock.patch.object(tronstore, 'parse_config', + return_value=(self.store_class, self.trans_method)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.Thread', + new=mock.Mock(return_value=self.mock_thread)), + mock.patch.object(signal, 'signal'), + mock.patch.object(tronstore, 'get_all_from_pipe', + side_effect=echo_requests), + mock.patch('tron.serialize.runstate.tronstore.tronstore.StoreRequestFactory', + new=mock.Mock(return_value=self.request_factory)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.StoreResponseFactory', + new=mock.Mock(return_value=self.response_factory)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.Lock', + new=mock.Mock(return_value=self.lock)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.Queue', + new=mock.Mock(return_value=self.queue)) + ) as ( + self.parse_patch, + self.thread_patch, + self.signal_patch, + self.get_all_patch, + self.request_patch, + self.response_patch, + self.lock_patch, + self.queue_patch + ): + yield + + def assert_main_startup(self): + self.parse_patch.assert_any_call(self.config) + self.request_patch.assert_called_once_with(self.trans_method) + self.response_patch.assert_called_once_with(self.trans_method) + assert_equal(self.lock_patch.call_count, 2) + self.queue_patch.assert_called_once_with() + self.thread_patch.assert_any_call(target=tronstore.thread_starter, args=(self.queue, [])) + self.mock_thread.start.assert_called_once_with() + self.signal_patch.assert_any_call(signal.SIGINT, tronstore.shutdown_handler) + self.signal_patch.assert_any_call(signal.SIGTERM, tronstore.shutdown_handler) + + def test_shutdown_request(self): + fake_id = 77 + self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_id)] + tronstore.main(self.config, self.pipe) + + self.assert_main_startup() + assert tronstore.is_shutdown + self.get_all_patch.assert_called_once_with(self.pipe) + self.store_class.cleanup.assert_called_once_with() + self.response_factory.build.assert_called_once_with(True, fake_id, '') + self.pipe.send_bytes.assert_called_once_with(self.response_factory.build().serialized) + + def test_config_request(self): + fake_id = 77 + fake_shutdown_id = 88 + self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), + mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] + tronstore.main(self.config, self.pipe) + + self.assert_main_startup() + assert tronstore.is_shutdown + assert_equal(self.store_class.cleanup.call_count, 2) + self.store_class.cleanup.assert_any_call() + assert_equal(self.parse_patch.call_count, 2) + self.request_factory.update_method.assert_called_once_with(self.trans_method) + self.response_factory.update_method.assert_called_once_with(self.trans_method) + + def test_save_request(self): + fake_id = 77 + fake_shutdown_id = 88 + self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SAVE, id=fake_id), + mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] + tronstore.main(self.config, self.pipe) + + self.assert_main_startup() + assert tronstore.is_shutdown + self.thread_patch.assert_any_call(target=tronstore.handle_request, + args=( + self.requests[0], + self.store_class, + self.pipe, + self.response_factory, + self.lock, + self.lock)) + self.queue.put.assert_called_once_with(self.mock_thread) + + def test_restore_request(self): + fake_id = 77 + fake_shutdown_id = 88 + self.requests = [mock.Mock(req_type=msg_enums.REQUEST_RESTORE, id=fake_id), + mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] + tronstore.main(self.config, self.pipe) + + self.assert_main_startup() + assert tronstore.is_shutdown + self.thread_patch.assert_any_call(target=tronstore.handle_request, + args=( + self.requests[0], + self.store_class, + self.pipe, + self.response_factory, + self.lock, + self.lock)) + self.queue.put.assert_called_once_with(self.mock_thread) + + +class TronstoreHandleRequestTestCase(TestCase): + + @setup + def setup_args(self): + self.store_class = mock.Mock() + self.pipe = mock.Mock() + self.factory = mock.Mock() + self.save_lock = mock.MagicMock() + self.restore_lock = mock.MagicMock() + + def test_handle_request_save(self): + fake_id = 3090 + request_data = ('fantastic', 'voyage') + data_type = 'lakeside' + fake_success = 'eaten_by_a_gru' + self.store_class.save = mock.Mock(return_value=fake_success) + request = mock.Mock(req_type=msg_enums.REQUEST_SAVE, data=request_data, + data_type=data_type, id=fake_id) + + tronstore.handle_request(request, self.store_class, self.pipe, + self.factory, self.save_lock, self.restore_lock) + + self.save_lock.__enter__.assert_called_once_with() + self.save_lock.__exit__.assert_called_once_with(None, None, None) + self.store_class.save.assert_called_once_with(request_data[0], request_data[1], + data_type) + self.factory.build.assert_called_once_with(fake_success, fake_id, '') + self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + + def test_handle_request_restore(self): + fake_id = 53045 + request_data = 'edgeworth' + fake_success = ('steel_samurai_fan', 'or_maybe_its_ironic') + data_type = 'lawyer' + self.store_class.restore = mock.Mock(return_value=fake_success) + request = mock.Mock(req_type=msg_enums.REQUEST_RESTORE, data=request_data, + data_type=data_type, id=fake_id) + + tronstore.handle_request(request, self.store_class, self.pipe, + self.factory, self.save_lock, self.restore_lock) + + self.restore_lock.__enter__.assert_called_once_with() + self.restore_lock.__exit__.assert_called_once_with(None, None, None) + self.store_class.restore.assert_called_once_with(request_data, data_type) + self.factory.build.assert_called_once_with(fake_success[0], fake_id, fake_success[1]) + self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + + def test_handle_request_other(self): + fake_id = 1234567890 + request = mock.Mock(req_type='not_actually_a_request', id=fake_id) + + tronstore.handle_request(request, self.store_class, self.pipe, + self.factory, self.save_lock, self.restore_lock) + + self.factory.build.assert_called_once_with(False, fake_id, '') + self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + + +class TronstoreOtherTestCase(TestCase): + + def test_parse_config(self): + fake_config = mock.Mock( + name='yo_earl', + transport_method='what', + store_type='you\'re fired', + connection_details='HNNNNNNLLLLLGGG', + db_store_method='one_too_many_lines') + fake_store = 'lady_madonna' + with mock.patch.object(tronstore.store, 'build_store', + return_value=fake_store) as build_patch: + assert_equal(tronstore.parse_config(fake_config), (fake_store, fake_config.transport_method)) + build_patch.assert_called_once_with( + fake_config.name, + fake_config.store_type, + fake_config.connection_details, + fake_config.db_store_method) + + def test_get_all_from_pipe(self): + fake_data = 'fuego' + pipe = mock.Mock() + pipe.recv_bytes = mock.Mock(return_value=fake_data) + pipe.poll = mock.Mock(side_effect=iter([True, False])) + assert_equal(tronstore.get_all_from_pipe(pipe), [fake_data]) + pipe.recv_bytes.assert_called_once_with() + assert_equal(pipe.poll.call_count, 2) + + +class TronstoreThreadStarterTestCase(TestCase): + + @setup_teardown + def setup_thread_starter(self): + tronstore.is_shutdown = False + with mock.patch.object(tronstore.time, 'sleep') as self.sleep_patch: + yield + + def test_pool_size_limit(self): + fake_thread = mock.Mock() + fake_queue = Queue() + map(fake_queue.put, [fake_thread for i in range(tronstore.POOL_SIZE + 1)]) + fake_thread.is_alive = lambda: not self.sleep_patch.called + running_threads = [] + + def shutdown_tronstore(time): + tronstore.is_shutdown = True + self.sleep_patch.configure_mock(side_effect=shutdown_tronstore) + + tronstore.thread_starter(fake_queue, running_threads) + assert_equal(running_threads, []) + assert_equal(fake_thread.start.call_count, tronstore.POOL_SIZE+1) + self.sleep_patch.assert_called_once_with(0.5) + assert fake_queue.empty() + tronstore.is_shutdown = False # to make sure nothing weird happens + + def test_shutdown_condition(self): + fake_queue = Queue() + running_threads = [] + tronstore.is_shutdown = True + + with mock.patch.object(tronstore, '_remove_finished_threads', + return_value=0) as remove_patch: + tronstore.thread_starter(fake_queue, running_threads) + assert not remove_patch.called + assert not self.sleep_patch.called + assert_equal(running_threads, []) + tronstore.is_shutdown = False + + def test_running_thread_operations(self): + fake_queue = Queue() + fake_thread = mock.Mock() + running_threads = mock.Mock(__len__=lambda i: 0) + + def shutdown_tronstore(time): + tronstore.is_shutdown = True + running_threads.append = mock.Mock(side_effect=shutdown_tronstore) + + tronstore.is_shutdown = False + fake_queue.put(fake_thread) + + with mock.patch.object(tronstore, '_remove_finished_threads', + return_value=0) as remove_patch: + tronstore.thread_starter(fake_queue, running_threads) + remove_patch.assert_called_once_with(running_threads) + fake_thread.start.assert_called_once_with() + running_threads.append.assert_called_once_with(fake_thread) + + def test_remove_finished_threads(self): + fake_thread = mock.Mock(is_alive=mock.Mock(return_value=False)) + fake_get = mock.Mock(return_value=fake_thread) + fake_pop = mock.Mock() + fake_len = 5 + running_threads = mock.Mock( + __len__=lambda i: fake_len, + __getitem__=fake_get, + pop=fake_pop) + assert_equal(tronstore._remove_finished_threads(running_threads), 5) + calls = [mock.call(i) for i in range(fake_len-1, -1, -1)] + fake_get.assert_has_calls(calls) + fake_pop.assert_has_calls(calls) + assert_equal(fake_thread.is_alive.call_count, 5) diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index c3a1ca21c..80b8900b0 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -22,13 +22,19 @@ from tron.serialize.runstate.tronstore import store from tron.serialize.runstate.tronstore import msg_enums +# This timeout MUST BE SHORTER than the one in process.py! +# Seriously, if this is longer, everything will break! +SHUTDOWN_TIMEOUT = 3.0 +POOL_SIZE = 35 + + def shutdown_handler(signum, frame): """This is just here to stop tronstore from exiting early. The process will be terminated from the main tron daemon when all requests have been finished. This is needed because Python propogates signals to spawned processes, and tronstore is going to get a TON of requests whenever a SIGINT is sent to the main daemon, as it has to save everything before - it can shut down.""" + it can gracefully shut down.""" pass @@ -42,30 +48,6 @@ def parse_config(config): return (store.build_store(name, store_type, connection_details, db_store_method), transport_method) -# def enqueue_input(stdin, queue): -# """Enqueue anything read by stdin into a queue. This is run in a separate -# thread such that requests can be processed without blocking.""" -# open('tronstore_is_running.test', 'w') -# try: -# for line in stdin: -# open('tronstore_got_a_line.test', 'w') -# f.write('I read a line!') -# f.close() -# stdin.close() -# except Exception, e: # something happened, finish up and exit -# open('tronstore_stdin_crashed.test', 'w') -# stdin.close() -# global is_shutdown -# is_shutdown = True - -# def start_stdin_thread(): -# """Starts the thread that will enqueue all stdin data into the stdin_queue.""" -# stdin_queue = Queue() -# stdin_thread = Thread(target=enqueue_input, args=(sys.stdin, stdin_queue)) -# stdin_thread.daemon = True -# stdin_thread.start() -# return stdin_queue - def get_all_from_pipe(pipe): """Gets all of the requests from the pipe, returning an array of serialized requests (they still need to be decoded). @@ -94,6 +76,20 @@ def handle_request(request, store_class, pipe, factory, save_lock, restore_lock) else: pipe.send_bytes(factory.build(False, request.id, '').serialized) + +def _remove_finished_threads(running_threads): + """A small helper function to clean out the running_threads array. + Doesn't actually create a new instance of a list; it modifies + the existing list as a side effect, and returns the number + of running threads that it cleaned up.""" + counter = 0 + for i in range(len(running_threads) - 1, -1, -1): + if not running_threads[i].is_alive(): + running_threads.pop(i) + counter += 1 + return counter + + def thread_starter(queue, running_threads): """A method to start threads that have been queued up in queue. Also takes a reference to a list (running_threads) that this function will store any @@ -105,20 +101,10 @@ def thread_starter(queue, running_threads): allow the main thread to know what's running. As such, all operations on running_threads must be method calls to modify the list instance given to this thread.""" - global is_shutdown - POOL_SIZE = 35 + global is_shutdown, POOL_SIZE pool_counter = POOL_SIZE - def _remove_finished_threads(running_threads): - # A small helper function to clean out the running_threads array. - counter = 0 - for thread in running_threads: - if not thread.is_alive(): - running_threads.remove(thread) - counter += 1 - return counter - while not is_shutdown or not queue.empty(): pool_counter += _remove_finished_threads(running_threads) @@ -144,10 +130,7 @@ def main(config, pipe): waits for requests to handle from pipe. It spawns threads for save and restore requests, which will send responses back over the pipe once completed.""" - global is_shutdown - # This timeout MUST BE SHORTER than the one in process.py! - # Seriously, if this is longer, everything will break! - SHUTDOWN_TIMEOUT = 3.0 + global is_shutdown, SHUTDOWN_TIMEOUT is_shutdown = False store_class, transport_method = parse_config(config) From c94d8c1f548b8f907c919567a31097201d2ef03b Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 18 Jul 2013 12:14:11 -0700 Subject: [PATCH 11/48] slight bugfix to solve race conditions in the migration script --- .../migrate_state_from_0.6.1_to_0.6.2.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index 9869a640b..d2eb9c5f4 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -133,6 +133,19 @@ def compile_new_info(options, state_info, new_file): return new_state_info +def assert_copied(new_store, data, key): + """A small function to counter race conditions. It's possible that + tronstore will serve the restore request BEFORE the save request, which + will result in an Exception. We simply retry 5 times (which should be more + than enough time for tronstore to serve the save request).""" + for i in range(5): + try: + assert data == new_store.restore([key])[key] + return + except Exception, e: + continue + raise AssertionError('The value %s failed to copy.' % key.iden) + def copy_metadata(old_store, new_store): meta_key_old = old_store.build_key(runstate.MCP_STATE, StateMetadata.name) old_metadata_dict = old_store.restore([meta_key_old]) @@ -140,7 +153,7 @@ def copy_metadata(old_store, new_store): old_metadata = old_metadata_dict[meta_key_old] meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) - assert old_metadata == new_store.restore([meta_key_new])[meta_key_new] + assert_copied(new_store, old_metadata, meta_key_new) def copy_services(old_store, new_store, service_names): for service in service_names: @@ -150,7 +163,7 @@ def copy_services(old_store, new_store, service_names): old_service_data = old_service_dict[service_key_old] service_key_new = new_store.build_key(runstate.SERVICE_STATE, service) new_store.save([(service_key_new, old_service_data)]) - assert old_service_data == new_store.restore([service_key_new])[service_key_new] + assert_copied(new_store, old_service_data, service_key_new) def copy_jobs(old_store, new_store, job_names): for job in job_names: @@ -166,12 +179,12 @@ def copy_jobs(old_store, new_store, job_names): job_run_key = new_store.build_key(runstate.JOB_RUN_STATE, job + ('.%s' % job_run['run_num'])) new_store.save([(job_run_key, job_run)]) - assert job_run == new_store.restore([job_run_key])[job_run_key] + assert_copied(new_store, job_run, job_run_key) run_ids = sorted(run_ids, reverse=True) job_state_data = {'enabled': old_job_data['enabled'], 'run_ids': run_ids} new_store.save([(job_state_key, job_state_data)]) - assert job_state_data == new_store.restore([job_state_key])[job_state_key] + assert_copied(new_store, job_state_data, job_state_key) def main(): @@ -193,7 +206,8 @@ def main(): copy_services(old_store, new_store, config.get_services().keys()) print('Converting job data...') copy_jobs(old_store, new_store, config.get_jobs().keys()) - print('...done.') + print('Done copying. All data has been verified.') + print('Cleaning up, just a sec...') old_store.cleanup() new_store.cleanup() From ada38df25d3628c99545c39363acb79d55883953 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 18 Jul 2013 14:57:20 -0700 Subject: [PATCH 12/48] merge of release_0.6.2, fix to default state persistence --- tron/config/config_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 9acd03a41..f664c1d77 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -438,7 +438,7 @@ def validate_jobs_and_services(config, config_context): config_utils.unique_names(fmt_string, config['jobs'], config['services']) -DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', 'pickle', None, 1, None) +DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', 'pickle', 1, 'json', 'pickle') DEFAULT_NODE = ValidateNode().do_shortcut('localhost') From a513345ee3997a8433fab292116d81a0be24a57f Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 18 Jul 2013 17:13:40 -0700 Subject: [PATCH 13/48] updated migrate_state.py, slight cleanup, enabled parallelstore in tron --- .../runstate/tronstore/parallelstore_test.py | 14 +-- .../runstate/tronstore/process_test.py | 4 +- tools/migration/migrate_state.py | 9 +- tools/migration/migrate_state_pre_0_6_2.py | 106 ++++++++++++++++++ tron/serialize/runstate/statemanager.py | 56 ++++++--- .../runstate/tronstore/parallelstore.py | 3 +- tron/serialize/runstate/tronstore/process.py | 3 +- .../serialize/runstate/tronstore/tronstore.py | 78 +++++++------ 8 files changed, 197 insertions(+), 76 deletions(-) create mode 100644 tools/migration/migrate_state_pre_0_6_2.py diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py index e4cdeb4f7..4b673a42e 100644 --- a/tests/serialize/runstate/tronstore/parallelstore_test.py +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -24,19 +24,7 @@ def setup_store(self): yield def test__init__(self): - self.process_patch.assert_called_once_with(self.store.path, self.config, self.store.response_factory) - - # def test_start_process(self): - # self.spawn_patch.assert_called_once_with( - # self.store.process, - # "serialize/runstate/tronstore/tronstore", - # ["tronstore", - # self.config.name, - # self.config.transport_method, - # self.config.store_type, - # self.config.connection_details, - # self.config.db_store_method]) - # self.run_patch.assert_called_once_with() + self.process_patch.assert_called_once_with(self.config, self.store.response_factory) def test_build_key(self): key_type = runstate.JOB_STATE diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index a305bcf92..6a9aa8709 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -25,16 +25,14 @@ def setup_process(self): db_store_method=None, buffer_size=1 ) - self.path = 'gaben/at/valve/software/dot/com' self.factory = mock.Mock() - self.process = StoreProcessProtocol(self.path, self.config, self.factory) + self.process = StoreProcessProtocol(self.config, self.factory) yield def test__init__(self): assert_equal(self.process.response_factory, self.factory) assert_equal(self.process.config, self.config) assert_equal(self.process.orphaned_responses, {}) - assert_equal(self.process.path, self.path) assert not self.process.is_shutdown def test_start_process(self): diff --git a/tools/migration/migrate_state.py b/tools/migration/migrate_state.py index 963f367c6..4be21e808 100644 --- a/tools/migration/migrate_state.py +++ b/tools/migration/migrate_state.py @@ -90,8 +90,11 @@ def convert_state(opts): job_states = add_namespaces(job_states) service_states = add_namespaces(service_states) - for name, job in job_states.iteritems(): - dest_manager.save(runstate.JOB_STATE, name, job) + for name, (job_state, run_list) in job_states.iteritems(): + dest_manager.save(runstate.JOB_STATE, name, job_state) + for run_data in run_list: + run_name = '%s.%s' % (run_data['job_name'], run_data['run_num']) + dest_manager.save(runstate.JOB_RUN_STATE, run_name, run_data) print "Migrated %s jobs." % len(job_states) for name, service in service_states.iteritems(): @@ -103,4 +106,4 @@ def convert_state(opts): if __name__ == "__main__": opts, _args = parse_options() - convert_state(opts) \ No newline at end of file + convert_state(opts) diff --git a/tools/migration/migrate_state_pre_0_6_2.py b/tools/migration/migrate_state_pre_0_6_2.py new file mode 100644 index 000000000..963f367c6 --- /dev/null +++ b/tools/migration/migrate_state_pre_0_6_2.py @@ -0,0 +1,106 @@ +""" + Migrate a state file/database from one StateStore implementation to another. It + may also be used to add namespace names to jobs/services when upgrading + from pre-0.5.2 to version 0.5.2. + + Usage: + python tools/migration/migrate_state.py \ + -s -d [ --namespace ] + + old_config.yaml and new_config.yaml should be configuration files with valid + state_persistence sections. The state_persistence section configures the + StateStore. + + Pre 0.5 state files can be read by the YamlStateStore. See the configuration + documentation for more details on how to create state_persistence sections. +""" +import optparse +from tron.config import manager, schema +from tron.serialize import runstate +from tron.serialize.runstate.statemanager import PersistenceManagerFactory +from tron.utils import tool_utils + + +def parse_options(): + parser = optparse.OptionParser() + parser.add_option('-s', '--source', + help="The source configuration path which contains a state_persistence " + "section configured for the state file/database.") + parser.add_option('-d', '--dest', + help="The destination configuration path which contains a " + "state_persistence section configured for the state file/database.") + parser.add_option('--source-working-dir', + help="The working directory for source dir to resolve relative paths.") + parser.add_option('--dest-working-dir', + help="The working directory for dest dir to resolve relative paths.") + parser.add_option('--namespace', action='store_true', + help="Move jobs/services which are missing a namespace to the MASTER") + + opts, args = parser.parse_args() + + if not opts.source: + parser.error("--source is required") + if not opts.dest: + parser.error("--dest is required.") + + return opts, args + + +def get_state_manager_from_config(config_path, working_dir): + """Return a state manager from the configuration. + """ + config_manager = manager.ConfigManager(config_path) + config_container = config_manager.load() + state_config = config_container.get_master().state_persistence + with tool_utils.working_dir(working_dir): + return PersistenceManagerFactory.from_config(state_config) + + +def get_current_config(config_path): + config_manager = manager.ConfigManager(config_path) + return config_manager.load() + + +def add_namespaces(state_data): + return dict(('%s.%s' % (schema.MASTER_NAMESPACE, name), data) + for (name, data) in state_data.iteritems()) + +def strip_namespace(names): + return [name.split('.', 1)[1] for name in names] + + +def convert_state(opts): + source_manager = get_state_manager_from_config(opts.source, opts.source_working_dir) + dest_manager = get_state_manager_from_config(opts.dest, opts.dest_working_dir) + container = get_current_config(opts.source) + + msg = "Migrating state from %s to %s" + print msg % (source_manager._impl, dest_manager._impl) + + job_names, service_names = container.get_job_and_service_names() + if opts.namespace: + job_names = strip_namespace(job_names) + service_names = strip_namespace(service_names) + + job_states, service_states = source_manager.restore( + job_names, service_names, skip_validation=True) + source_manager.cleanup() + + if opts.namespace: + job_states = add_namespaces(job_states) + service_states = add_namespaces(service_states) + + for name, job in job_states.iteritems(): + dest_manager.save(runstate.JOB_STATE, name, job) + print "Migrated %s jobs." % len(job_states) + + for name, service in service_states.iteritems(): + dest_manager.save(runstate.SERVICE_STATE, name, service) + print "Migrated %s services." % len(service_states) + + dest_manager.cleanup() + + +if __name__ == "__main__": + opts, _args = parse_options() + convert_state(opts) \ No newline at end of file diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index e4451eb89..d7a436d30 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -6,10 +6,11 @@ from tron.config import schema from tron.core import job, jobrun, service from tron.serialize import runstate -from tron.serialize.runstate.mongostore import MongoStateStore -from tron.serialize.runstate.shelvestore import ShelveStateStore -from tron.serialize.runstate.sqlalchemystore import SQLAlchemyStateStore -from tron.serialize.runstate.yamlstore import YamlStateStore +# from tron.serialize.runstate.mongostore import MongoStateStore +# from tron.serialize.runstate.shelvestore import ShelveStateStore +# from tron.serialize.runstate.sqlalchemystore import SQLAlchemyStateStore +# from tron.serialize.runstate.yamlstore import YamlStateStore +from tron.serialize.runstate.tronstore.parallelstore import ParallelStore from tron.utils import observer log = logging.getLogger(__name__) @@ -24,30 +25,39 @@ class PersistenceStoreError(ValueError): class PersistenceManagerFactory(object): """Create a PersistentStateManager.""" + # TODO: Remove this class, it's somewhat pointless now @classmethod def from_config(cls, persistence_config): store_type = persistence_config.store_type - name = persistence_config.name - connection_details = persistence_config.connection_details + transport_method = persistence_config.transport_method + db_store_method = persistence_config.db_store_method buffer_size = persistence_config.buffer_size - store = None + # name = persistence_config.name + # connection_details = persistence_config.connection_details if store_type not in schema.StatePersistenceTypes: raise PersistenceStoreError("Unknown store type: %s" % store_type) - if store_type == schema.StatePersistenceTypes.shelve: - store = ShelveStateStore(name) + if transport_method not in schema.StateTransportTypes: + raise PersistenceStoreError("Unknown transport type: %s" % transport_method) - if store_type == schema.StatePersistenceTypes.sql: - store = SQLAlchemyStateStore(name, connection_details) + if db_store_method not in schema.StateTransportTypes and store_type in ('sql', 'mongo'): + raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) - if store_type == schema.StatePersistenceTypes.mongo: - store = MongoStateStore(name, connection_details) + # if store_type == schema.StatePersistenceTypes.shelve: + # store = ShelveStateStore(name) - if store_type == schema.StatePersistenceTypes.yaml: - store = YamlStateStore(name) + # if store_type == schema.StatePersistenceTypes.sql: + # store = SQLAlchemyStateStore(name, connection_details) + # if store_type == schema.StatePersistenceTypes.mongo: + # store = MongoStateStore(name, connection_details) + + # if store_type == schema.StatePersistenceTypes.yaml: + # store = YamlStateStore(name) + + store = ParallelStore(persistence_config) buffer = StateSaveBuffer(buffer_size) return PersistentStateManager(store, buffer) @@ -127,10 +137,14 @@ def restore(self, keys): def save(self, key, state_data): pass + def load_config(self, new_config): + pass + def cleanup(self): pass """ + # TODO: Rename things here, as ParallelStore is always used def __init__(self, persistence_impl, buffer): self.enabled = True @@ -200,6 +214,10 @@ def _save_from_buffer(self): log.warn(msg) raise PersistenceStoreError(msg) + def update_from_config(self, new_state_config): + self._save_from_buffer() + self._impl.load_config(new_state_config) + def cleanup(self): self._save_from_buffer() self._impl.cleanup() @@ -251,8 +269,12 @@ def update_from_config(self, state_config): if self.config == state_config: return False - self.shutdown() - self.state_manager = PersistenceManagerFactory.from_config(state_config) + if self.state_manager is NullStateManager: + self.state_manager = PersistenceManagerFactory.from_config(state_config) + # self.shutdown() + # self.state_manager = PersistenceManagerFactory.from_config(state_config) + else: + self.state_manager.update_from_config(state_config) self.config = state_config return True diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index cdae280d5..ce03049f4 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -42,8 +42,7 @@ class ParallelStore(object): def __init__(self, config): self.request_factory = StoreRequestFactory(config.transport_method) self.response_factory = StoreResponseFactory(config.transport_method) - self.path = os.path.dirname(msg_enums.__file__) + "/tronstore" - self.process = StoreProcessProtocol(self.path, config, self.response_factory) + self.process = StoreProcessProtocol(config, self.response_factory) def build_key(self, type, iden): return ParallelKey(type, iden) diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index dcb2a99b0..e9cfcc8e4 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -29,11 +29,10 @@ class StoreProcessProtocol(object): SHUTDOWN_TIMEOUT = 100.0 POLL_TIMEOUT = 10.0 - def __init__(self, path, config, response_factory): + def __init__(self, config, response_factory): self.config = config self.response_factory = response_factory self.orphaned_responses = {} - self.path = path self.is_shutdown = False self._start_process() diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 80b8900b0..9e03a8a8c 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -140,46 +140,52 @@ def main(config, pipe): save_lock = Lock() restore_lock = Lock() + signal.signal(signal.SIGINT, shutdown_handler) + signal.signal(signal.SIGTERM, shutdown_handler) + running_threads = [] thread_queue = Queue() thread_pool = Thread(target=thread_starter, args=(thread_queue, running_threads)) + thread_pool.daemon = True thread_pool.start() - signal.signal(signal.SIGINT, shutdown_handler) - signal.signal(signal.SIGTERM, shutdown_handler) - while True: timeout = SHUTDOWN_TIMEOUT if is_shutdown else None - if pipe.poll(timeout): - requests = get_all_from_pipe(pipe) - requests = map(request_factory.rebuild, requests) - for request in requests: - if request.req_type == msg_enums.REQUEST_SHUTDOWN: - is_shutdown = True - shutdown_req_id = request.id - - elif request.req_type == msg_enums.REQUEST_CONFIG: - while len(running_threads) != 0: - time.sleep(0.5) - store_class.cleanup() - store_class, transport_method = parse_config(request.data) - request_factory.update_method(transport_method) - response_factory.update_method(transport_method) - - else: - request_thread = Thread(target=handle_request, - args=( - request, - store_class, - pipe, - response_factory, - save_lock, - restore_lock)) - thread_queue.put(request_thread) - else: - # We have to wait for all requests to clean up first. - while len(running_threads) != 0: - time.sleep(0.5) - store_class.cleanup() - pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) - return + try: + if pipe.poll(timeout): + requests = get_all_from_pipe(pipe) + requests = map(request_factory.rebuild, requests) + for request in requests: + if request.req_type == msg_enums.REQUEST_SHUTDOWN: + is_shutdown = True + shutdown_req_id = request.id + + elif request.req_type == msg_enums.REQUEST_CONFIG: + while len(running_threads) != 0: + time.sleep(0.5) + store_class.cleanup() + store_class, transport_method = parse_config(request.data) + request_factory.update_method(transport_method) + response_factory.update_method(transport_method) + + else: + request_thread = Thread(target=handle_request, + args=( + request, + store_class, + pipe, + response_factory, + save_lock, + restore_lock)) + request_thread.daemon = True + thread_queue.put(request_thread) + else: + # We have to wait for all requests to clean up first. + while len(running_threads) != 0: + time.sleep(0.5) + store_class.cleanup() + pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) + return + # Signals cause pipe.poll to throw IOErrors... + except IOError: + continue From 8806cc455221b415ff972a793a825218ebad7807 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 22 Jul 2013 11:08:09 -0700 Subject: [PATCH 14/48] tons of bugfixes, test fixes, new tests, etc --- tests/mcp_reconfigure_test.py | 1 + tests/serialize/runstate/statemanager_test.py | 75 +++++++++++++++---- .../runstate/tronstore/process_test.py | 10 ++- .../runstate/tronstore/tronstore_test.py | 28 ++++--- tron/serialize/runstate/statemanager.py | 2 - tron/serialize/runstate/tronstore/process.py | 31 +++++++- .../serialize/runstate/tronstore/tronstore.py | 50 +++++++------ tron/trondaemon.py | 4 +- 8 files changed, 143 insertions(+), 58 deletions(-) diff --git a/tests/mcp_reconfigure_test.py b/tests/mcp_reconfigure_test.py index 3ad3b1316..184031d6c 100644 --- a/tests/mcp_reconfigure_test.py +++ b/tests/mcp_reconfigure_test.py @@ -143,6 +143,7 @@ def teardown_mcp(self): event.EventManager.reset() filehandler.OutputPath(self.test_dir).delete() filehandler.FileHandleManager.reset() + self.mcp.state_watcher.shutdown() def reconfigure(self): config = {schema.MASTER_NAMESPACE: self._get_config(1, self.test_dir)} diff --git a/tests/serialize/runstate/statemanager_test.py b/tests/serialize/runstate/statemanager_test.py index f6e45999a..5d9848f9b 100644 --- a/tests/serialize/runstate/statemanager_test.py +++ b/tests/serialize/runstate/statemanager_test.py @@ -1,6 +1,7 @@ import os import mock -from testify import TestCase, assert_equal, setup, run +import contextlib +from testify import TestCase, assert_equal, setup, run, setup_teardown from tests.assertions import assert_raises from tests.testingutils import autospec_method @@ -13,22 +14,59 @@ from tron.serialize.runstate.statemanager import PersistenceStoreError from tron.serialize.runstate.statemanager import VersionMismatchError from tron.serialize.runstate.statemanager import PersistenceManagerFactory +from tron.serialize.runstate.statemanager import NullStateManager class PersistenceManagerFactoryTestCase(TestCase): - def test_from_config_shelve(self): - thefilename = 'thefilename' - config = schema.ConfigState( - store_type='shelve', name=thefilename, buffer_size=0, - transport_method='pickle', - connection_details=None, - db_store_method=None) - manager = PersistenceManagerFactory.from_config(config) - store = manager._impl - assert_equal(store.filename, config.name) - assert isinstance(store, ShelveStateStore) - os.unlink(thefilename) + @setup_teardown + def setup_factory_and_enumerate(self): + self.mock_buffer_size = 25 + self.mock_config = mock.Mock(buffer_size=self.mock_buffer_size) + with contextlib.nested( + mock.patch('tron.serialize.runstate.statemanager.ParallelStore', + autospec=True), + mock.patch('%s.%s' % (PersistentStateManager.__module__, PersistentStateManager.__name__), + autospec=True), + mock.patch('%s.%s' % (StateSaveBuffer.__module__, StateSaveBuffer.__name__), + autospec=True) + ) as (self.parallel_patch, self.state_patch, self.buffer_patch): + yield + + def test_from_config_all_valid_enum_types(self): + for store_type in schema.StatePersistenceTypes.values: + self.mock_config.configure_mock(store_type=store_type) + for transport_method in schema.StateTransportTypes.values: + self.mock_config.configure_mock(transport_method=transport_method) + if store_type in ('sql', 'mongo'): + self.mock_config.configure_mock(db_store_method=transport_method) + + assert_equal(PersistenceManagerFactory.from_config(self.mock_config), + self.state_patch(self.parallel_patch, self.buffer_patch)) + self.parallel_patch.assert_called_with(self.mock_config) + self.buffer_patch.assert_called_with(self.mock_buffer_size) + + def test_from_config_invalid_store_type(self): + self.mock_config.configure_mock(store_type='play_the_game') + for transport_method in schema.StateTransportTypes.values: + self.mock_config.configure_mock(transport_method=transport_method) + assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) + + def test_from_config_invalid_transport_type(self): + self.mock_config.configure_mock(transport_method='ghosts_cant_eat') + for store_type in schema.StatePersistenceTypes.values: + self.mock_config.configure_mock(store_type=store_type) + if store_type in ('sql', 'mongo'): + self.mock_config.configure_mock(db_store_method='json') + assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) + + def test_from_config_invalid_db_store_method(self): + self.mock_config.configure_mock(db_store_method='im_running_out_of_strs') + for store_type in ('sql', 'mongo'): + self.mock_config.configure_mock(store_type=store_type) + for transport_method in schema.StateTransportTypes.values: + self.mock_config.configure_mock(transport_method=transport_method) + assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) class StateMetadataTestCase(TestCase): @@ -157,16 +195,21 @@ def test_update_from_config_no_change(self): @mock.patch('tron.serialize.runstate.statemanager.PersistenceManagerFactory', autospec=True) - def test_update_from_config_changed(self, mock_factory): + def test_update_from_config_no_state_manager(self, mock_factory): state_config = mock.Mock() - autospec_method(self.watcher.shutdown) + self.watcher.state_manager = NullStateManager assert self.watcher.update_from_config(state_config) assert_equal(self.watcher.config, state_config) - self.watcher.shutdown.assert_called_with() assert_equal(self.watcher.state_manager, mock_factory.from_config.return_value) mock_factory.from_config.assert_called_with(state_config) + def test_update_from_config_with_state_manager(self): + state_config = mock.Mock() + assert self.watcher.update_from_config(state_config) + assert_equal(self.watcher.config, state_config) + self.state_manager.update_from_config.assert_called_once_with(state_config) + def test_save_job(self): mock_job = mock.Mock() self.watcher.save_job(mock_job) diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index 6a9aa8709..805094565 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -1,6 +1,8 @@ import contextlib import mock -from testify import TestCase, run, assert_equal, assert_raises, setup_teardown +import signal +import os +from testify import TestCase, assert_equal, assert_raises, setup_teardown from tron.serialize.runstate.tronstore import tronstore from tron.serialize.runstate.tronstore.process import StoreProcessProtocol, TronStoreError @@ -139,15 +141,15 @@ def test_send_request_shutdown_not_shutdown(self): mock.patch.object(self.process.pipe, 'close'), mock.patch.object(self.process.pipe, 'send_bytes'), mock.patch.object(self.process, '_poll_for_response', return_value=mock.Mock()), - mock.patch.object(self.process.process, 'terminate') - ) as (alive_patch, close_patch, send_patch, poll_patch, terminate_patch): + mock.patch.object(os, 'kill') + ) as (alive_patch, close_patch, send_patch, poll_patch, kill_patch): self.process.send_request_shutdown(test_request) alive_patch.assert_called_once_with() assert self.process.is_shutdown send_patch.assert_called_once_with(test_request.serialized) poll_patch.assert_called_once_with(fake_id, self.process.SHUTDOWN_TIMEOUT) close_patch.assert_called_once_with() - terminate_patch.assert_called_once_with() + kill_patch.assert_called_once_with(self.process.process.pid, signal.SIGKILL) def test_send_request_shutdown_is_shutdown(self): self.process.is_shutdown = True diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py index c05d1607d..fd9a87134 100644 --- a/tests/serialize/runstate/tronstore/tronstore_test.py +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -14,14 +14,14 @@ def setup_main(self): self.pipe = mock.Mock() self.store_class = mock.Mock() self.trans_method = mock.Mock() - self.mock_thread = mock.Mock() + self.mock_thread = mock.Mock(is_alive=lambda: False) self.request_factory = mock.Mock() self.response_factory = mock.Mock() self.lock = mock.Mock() self.queue = mock.Mock() def poll_patch(timeout): - return False if timeout else True + return False if tronstore.is_shutdown else True self.pipe.poll = poll_patch def echo_single_request(request): @@ -31,6 +31,9 @@ def echo_single_request(request): def echo_requests(not_used): return self.requests + def raise_to_exit(exitcode): + raise SystemError + with contextlib.nested( mock.patch.object(tronstore, 'parse_config', return_value=(self.store_class, self.trans_method)), @@ -46,7 +49,9 @@ def echo_requests(not_used): mock.patch('tron.serialize.runstate.tronstore.tronstore.Lock', new=mock.Mock(return_value=self.lock)), mock.patch('tron.serialize.runstate.tronstore.tronstore.Queue', - new=mock.Mock(return_value=self.queue)) + new=mock.Mock(return_value=self.queue)), + mock.patch.object(tronstore.os, '_exit', + side_effect=raise_to_exit) ) as ( self.parse_patch, self.thread_patch, @@ -55,7 +60,8 @@ def echo_requests(not_used): self.request_patch, self.response_patch, self.lock_patch, - self.queue_patch + self.queue_patch, + self.exit_patch ): yield @@ -67,13 +73,15 @@ def assert_main_startup(self): self.queue_patch.assert_called_once_with() self.thread_patch.assert_any_call(target=tronstore.thread_starter, args=(self.queue, [])) self.mock_thread.start.assert_called_once_with() - self.signal_patch.assert_any_call(signal.SIGINT, tronstore.shutdown_handler) - self.signal_patch.assert_any_call(signal.SIGTERM, tronstore.shutdown_handler) + self.signal_patch.assert_any_call(signal.SIGINT, tronstore._discard_signal) + self.signal_patch.assert_any_call(signal.SIGHUP, tronstore._discard_signal) + self.signal_patch.assert_any_call(signal.SIGTERM, tronstore._discard_signal) + self.exit_patch.assert_called_once_with(0) def test_shutdown_request(self): fake_id = 77 self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_id)] - tronstore.main(self.config, self.pipe) + assert_raises(SystemError, tronstore.main, self.config, self.pipe) self.assert_main_startup() assert tronstore.is_shutdown @@ -87,7 +95,7 @@ def test_config_request(self): fake_shutdown_id = 88 self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - tronstore.main(self.config, self.pipe) + assert_raises(SystemError, tronstore.main, self.config, self.pipe) self.assert_main_startup() assert tronstore.is_shutdown @@ -102,7 +110,7 @@ def test_save_request(self): fake_shutdown_id = 88 self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SAVE, id=fake_id), mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - tronstore.main(self.config, self.pipe) + assert_raises(SystemError, tronstore.main, self.config, self.pipe) self.assert_main_startup() assert tronstore.is_shutdown @@ -121,7 +129,7 @@ def test_restore_request(self): fake_shutdown_id = 88 self.requests = [mock.Mock(req_type=msg_enums.REQUEST_RESTORE, id=fake_id), mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - tronstore.main(self.config, self.pipe) + assert_raises(SystemError, tronstore.main, self.config, self.pipe) self.assert_main_startup() assert tronstore.is_shutdown diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index d7a436d30..20573a21a 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -271,8 +271,6 @@ def update_from_config(self, state_config): if self.state_manager is NullStateManager: self.state_manager = PersistenceManagerFactory.from_config(state_config) - # self.shutdown() - # self.state_manager = PersistenceManagerFactory.from_config(state_config) else: self.state_manager.update_from_config(state_config) self.config = state_config diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index e9cfcc8e4..4ebedfdff 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -1,6 +1,7 @@ -import time +# import time +import signal import logging -# import os +import os from multiprocessing import Process, Pipe from tron.serialize.runstate.tronstore import tronstore @@ -43,6 +44,12 @@ def _start_process(self): self.pipe, child_pipe = Pipe() store_args = (self.config, child_pipe) + # See the long comment in tronstore.main() as to why we have to do this + # if not signal.getsignal(signal.SIGCHLD): + # print('registering child signal handler') + # signal.signal(signal.SIGCHLD, signal.getsignal(signal.SIGTERM)) + # print('registered to %s' % signal.getsignal(signal.SIGCHLD)) + self.process = Process(target=tronstore.main, args=store_args) self.process.daemon = True self.process.start() @@ -127,10 +134,26 @@ def send_request_shutdown(self, request): response = self._poll_for_response(request.id, self.SHUTDOWN_TIMEOUT) if not response or not response.success: - log.error("tronstore failed to shut down successfully.") + log.error("tronstore failed to shut down cleanly.") self.pipe.close() - self.process.terminate() + # We can't actually use process.terminate(), as that sends a SIGTERM + # to the process, which unfortunately is registered to call the same + # handler as trond due to how Python copies its environment over + # to new processes. In addition, using process.terminate causes + # SIGTERMs to get sent to everything that process spawns in its + # route to shutting down- and when the trond event handler calls some + # stuff that tronstore would never actually touch, and these calls + # start some initialization related call stacks, this results in a + # bunch of really strange call stacks all getting SIGTERMs that ALL end + # up calling the trond signal handler, ending in this horrible + # unclean shutdown. + # + # Using os.kill has the effect we actually want, which is to just + # kill tronstore completely. We can do this safely at this point, as + # tronstore ONLY sends the shutdown response when it's finished + # all of its requests and shutting down the store object. + os.kill(self.process.pid, signal.SIGKILL) def update_config(self, new_config, config_request): self.send_request(config_request) diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 9e03a8a8c..1f765c3fd 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -15,6 +15,7 @@ """ import time import signal +import os from threading import Thread, Lock from Queue import Queue, Empty @@ -24,17 +25,14 @@ # This timeout MUST BE SHORTER than the one in process.py! # Seriously, if this is longer, everything will break! -SHUTDOWN_TIMEOUT = 3.0 +SHUTDOWN_TIMEOUT = 1.0 +# this can be rather long- it's only real use it to clean up tronstore +# in case it's zombied +POLL_TIMEOUT = 2.0 POOL_SIZE = 35 -def shutdown_handler(signum, frame): - """This is just here to stop tronstore from exiting early. The process - will be terminated from the main tron daemon when all requests have been - finished. This is needed because Python propogates signals to - spawned processes, and tronstore is going to get a TON of requests whenever - a SIGINT is sent to the main daemon, as it has to save everything before - it can gracefully shut down.""" +def _discard_signal(signum, frame): pass @@ -130,8 +128,10 @@ def main(config, pipe): waits for requests to handle from pipe. It spawns threads for save and restore requests, which will send responses back over the pipe once completed.""" - global is_shutdown, SHUTDOWN_TIMEOUT + global is_shutdown, SHUTDOWN_TIMEOUT, POLL_TIMEOUT + is_shutdown = False + shutdown_req_id = None store_class, transport_method = parse_config(config) @@ -140,19 +140,19 @@ def main(config, pipe): save_lock = Lock() restore_lock = Lock() - signal.signal(signal.SIGINT, shutdown_handler) - signal.signal(signal.SIGTERM, shutdown_handler) + signal.signal(signal.SIGINT, _discard_signal) + signal.signal(signal.SIGHUP, _discard_signal) + signal.signal(signal.SIGTERM, _discard_signal) running_threads = [] thread_queue = Queue() thread_pool = Thread(target=thread_starter, args=(thread_queue, running_threads)) - thread_pool.daemon = True + thread_pool.daemon = False thread_pool.start() while True: - timeout = SHUTDOWN_TIMEOUT if is_shutdown else None try: - if pipe.poll(timeout): + if pipe.poll(POLL_TIMEOUT): requests = get_all_from_pipe(pipe) requests = map(request_factory.rebuild, requests) for request in requests: @@ -179,13 +179,21 @@ def main(config, pipe): restore_lock)) request_thread.daemon = True thread_queue.put(request_thread) - else: + elif is_shutdown: # We have to wait for all requests to clean up first. - while len(running_threads) != 0: + while len(running_threads) != 0 or thread_pool.is_alive(): time.sleep(0.5) store_class.cleanup() - pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) - return - # Signals cause pipe.poll to throw IOErrors... - except IOError: - continue + if shutdown_req_id: + pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) + os._exit(0) + else: + # Did tron die? + try: + os.kill(os.getppid(), 0) + except: + is_shutdown = True + except IOError, e: + # Error #4 is a system interrupt, caused by ^C + if e.errno != 4: + raise e diff --git a/tron/trondaemon.py b/tron/trondaemon.py index bdbb508f4..077637714 100644 --- a/tron/trondaemon.py +++ b/tron/trondaemon.py @@ -154,6 +154,7 @@ def _build_context(self, options, context_class): signal.SIGINT: self._handle_graceful_shutdown, signal.SIGTERM: self._handle_shutdown, } + pidfile = PIDFile(options.pid_file) return context_class( working_directory=options.working_dir, @@ -199,7 +200,8 @@ def _run_mcp(self): def _run_reactor(self): """Run the twisted reactor.""" - self.reactor.run() + # Not setting this flag caused me 9 painful hours of debugging =( + self.reactor.run(installSignalHandlers=0) def _handle_shutdown(self, sig_num, stack_frame): log.info("Shutdown requested: sig %s" % sig_num) From b353fd445c862e34f0fd1c16b2ca5d78418879d9 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 22 Jul 2013 11:13:15 -0700 Subject: [PATCH 15/48] comment cleanup --- .../runstate/tronstore/tronstore_test.py | 2 +- .../runstate/tronstore/parallelstore.py | 1 - tron/serialize/runstate/tronstore/process.py | 29 ++++--------------- .../serialize/runstate/tronstore/tronstore.py | 8 ++--- 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py index fd9a87134..6d21b576c 100644 --- a/tests/serialize/runstate/tronstore/tronstore_test.py +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -2,7 +2,7 @@ import mock import signal from Queue import Queue -from testify import TestCase, run, assert_equal, assert_raises, setup_teardown, setup +from testify import TestCase, assert_equal, assert_raises, setup_teardown, setup from tron.serialize.runstate.tronstore import tronstore, msg_enums diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index ce03049f4..2784938f3 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -3,7 +3,6 @@ import logging import os -# from twisted.internet import reactor from tron.serialize.runstate.tronstore.process import StoreProcessProtocol from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory from tron.serialize.runstate.tronstore import msg_enums diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index 4ebedfdff..bb52aa406 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -1,11 +1,9 @@ -# import time import signal import logging import os from multiprocessing import Process, Pipe from tron.serialize.runstate.tronstore import tronstore -# from tron.serialize.runstate.tronstore.chunking import StoreChunkHandler log = logging.getLogger(__name__) @@ -26,7 +24,7 @@ class StoreProcessProtocol(object): while requests have an enumerator (see msg_enums.py) to identify the type of request. """ - # This timeout MUST be longer than the one in tronstore! + # This timeout MUST be longer than the POLL_TIMEOUT in tronstore! SHUTDOWN_TIMEOUT = 100.0 POLL_TIMEOUT = 10.0 @@ -44,12 +42,6 @@ def _start_process(self): self.pipe, child_pipe = Pipe() store_args = (self.config, child_pipe) - # See the long comment in tronstore.main() as to why we have to do this - # if not signal.getsignal(signal.SIGCHLD): - # print('registering child signal handler') - # signal.signal(signal.SIGCHLD, signal.getsignal(signal.SIGTERM)) - # print('registered to %s' % signal.getsignal(signal.SIGCHLD)) - self.process = Process(target=tronstore.main, args=store_args) self.process.daemon = True self.process.start() @@ -138,21 +130,10 @@ def send_request_shutdown(self, request): self.pipe.close() # We can't actually use process.terminate(), as that sends a SIGTERM - # to the process, which unfortunately is registered to call the same - # handler as trond due to how Python copies its environment over - # to new processes. In addition, using process.terminate causes - # SIGTERMs to get sent to everything that process spawns in its - # route to shutting down- and when the trond event handler calls some - # stuff that tronstore would never actually touch, and these calls - # start some initialization related call stacks, this results in a - # bunch of really strange call stacks all getting SIGTERMs that ALL end - # up calling the trond signal handler, ending in this horrible - # unclean shutdown. - # - # Using os.kill has the effect we actually want, which is to just - # kill tronstore completely. We can do this safely at this point, as - # tronstore ONLY sends the shutdown response when it's finished - # all of its requests and shutting down the store object. + # to the process, which unfortunately is registered to do nothing + # (as the process depends on trond to shut itself down, and shuts + # itself down if trond is dead anyway.) + # We want a hard kill anyway. os.kill(self.process.pid, signal.SIGKILL) def update_config(self, new_config, config_request): diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 1f765c3fd..d4e5a4c0d 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -23,11 +23,9 @@ from tron.serialize.runstate.tronstore import store from tron.serialize.runstate.tronstore import msg_enums -# This timeout MUST BE SHORTER than the one in process.py! -# Seriously, if this is longer, everything will break! -SHUTDOWN_TIMEOUT = 1.0 # this can be rather long- it's only real use it to clean up tronstore -# in case it's zombied +# in case it's zombied... however, it should be SHORTER than +# SHUTDOWN_TIMEOUT in process.py POLL_TIMEOUT = 2.0 POOL_SIZE = 35 @@ -128,7 +126,7 @@ def main(config, pipe): waits for requests to handle from pipe. It spawns threads for save and restore requests, which will send responses back over the pipe once completed.""" - global is_shutdown, SHUTDOWN_TIMEOUT, POLL_TIMEOUT + global is_shutdown, POLL_TIMEOUT is_shutdown = False shutdown_req_id = None From 9af7ef86d72e40b63a126c45cf98ed74d3b5b230 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 22 Jul 2013 11:17:43 -0700 Subject: [PATCH 16/48] a bit more comment cleanup --- tron/serialize/runstate/statemanager.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 20573a21a..09cbeff4c 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -33,8 +33,6 @@ def from_config(cls, persistence_config): transport_method = persistence_config.transport_method db_store_method = persistence_config.db_store_method buffer_size = persistence_config.buffer_size - # name = persistence_config.name - # connection_details = persistence_config.connection_details if store_type not in schema.StatePersistenceTypes: raise PersistenceStoreError("Unknown store type: %s" % store_type) @@ -45,18 +43,6 @@ def from_config(cls, persistence_config): if db_store_method not in schema.StateTransportTypes and store_type in ('sql', 'mongo'): raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) - # if store_type == schema.StatePersistenceTypes.shelve: - # store = ShelveStateStore(name) - - # if store_type == schema.StatePersistenceTypes.sql: - # store = SQLAlchemyStateStore(name, connection_details) - - # if store_type == schema.StatePersistenceTypes.mongo: - # store = MongoStateStore(name, connection_details) - - # if store_type == schema.StatePersistenceTypes.yaml: - # store = YamlStateStore(name) - store = ParallelStore(persistence_config) buffer = StateSaveBuffer(buffer_size) return PersistentStateManager(store, buffer) From b1354290fbb63afb24408f24d9614a38e539da24 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 22 Jul 2013 14:56:14 -0700 Subject: [PATCH 17/48] fixed up tronstore reconfiguration, new unit tests to reflect this --- tests/serialize/runstate/statemanager_test.py | 19 ++++++++++++- .../runstate/tronstore/parallelstore_test.py | 27 ++++++++++++++++-- .../runstate/tronstore/process_test.py | 7 ++--- .../runstate/tronstore/tronstore_test.py | 28 ++++++++++++++++++- tron/serialize/runstate/statemanager.py | 6 ++-- .../runstate/tronstore/parallelstore.py | 11 ++++++-- tron/serialize/runstate/tronstore/process.py | 12 ++++---- .../serialize/runstate/tronstore/tronstore.py | 20 +++++++++---- 8 files changed, 104 insertions(+), 26 deletions(-) diff --git a/tests/serialize/runstate/statemanager_test.py b/tests/serialize/runstate/statemanager_test.py index 5d9848f9b..3586fcd07 100644 --- a/tests/serialize/runstate/statemanager_test.py +++ b/tests/serialize/runstate/statemanager_test.py @@ -177,6 +177,15 @@ def test_disabled_nested(self): pass assert not self.manager.enabled + def test_update_config(self): + new_config = mock.Mock() + fake_return = 'reelin_in_the_years' + self.store.load_config.configure_mock(return_value=fake_return) + with mock.patch.object(self.manager, '_save_from_buffer') as save_patch: + assert_equal(self.manager.update_from_config(new_config), fake_return) + save_patch.assert_called_once_with() + self.store.load_config.assert_called_once_with(new_config) + class StateChangeWatcherTestCase(TestCase): @@ -204,12 +213,20 @@ def test_update_from_config_no_state_manager(self, mock_factory): mock_factory.from_config.return_value) mock_factory.from_config.assert_called_with(state_config) - def test_update_from_config_with_state_manager(self): + def test_update_from_config_with_state_manager_success(self): state_config = mock.Mock() assert self.watcher.update_from_config(state_config) assert_equal(self.watcher.config, state_config) self.state_manager.update_from_config.assert_called_once_with(state_config) + def test_update_from_config_failure(self): + self.state_manager.update_from_config.configure_mock(return_value=False) + state_config = self.watcher.config + fake_config = mock.Mock() + assert not self.watcher.update_from_config(fake_config) + assert_equal(self.watcher.config, state_config) + self.state_manager.update_from_config.assert_called_once_with(fake_config) + def test_save_job(self): mock_job = mock.Mock() self.watcher.save_job(mock_job) diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py index 4b673a42e..9139c119c 100644 --- a/tests/serialize/runstate/tronstore/parallelstore_test.py +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -81,17 +81,38 @@ def test_cleanup(self): self.store.cleanup() clean_patch.assert_called_once_with() - def test_load_config(self): + def test_load_config_success(self): new_config = mock.Mock() config_req = mock.Mock() with contextlib.nested( mock.patch.object(self.store.request_factory, 'update_method'), mock.patch.object(self.store.response_factory, 'update_method'), mock.patch.object(self.store.process, 'update_config'), - mock.patch.object(self.store.request_factory, 'build', return_value=config_req) - ) as (request_patch, response_patch, update_patch, build_patch): + mock.patch.object(self.store.request_factory, 'build', return_value=config_req), + mock.patch.object(self.store.process, 'send_request_get_response', + return_value=mock.Mock(success=True)) + ) as (request_patch, response_patch, update_patch, build_patch, send_patch): self.store.load_config(new_config) build_patch.assert_called_once_with(msg_enums.REQUEST_CONFIG, '', new_config) + send_patch.assert_called_once_with(config_req) request_patch.assert_called_once_with(new_config.transport_method) response_patch.assert_called_once_with(new_config.transport_method) update_patch.assert_called_once_with(new_config, config_req) + + def test_load_config_failure(self): + new_config = mock.Mock() + config_req = mock.Mock() + with contextlib.nested( + mock.patch.object(self.store.request_factory, 'update_method'), + mock.patch.object(self.store.response_factory, 'update_method'), + mock.patch.object(self.store.process, 'update_config'), + mock.patch.object(self.store.request_factory, 'build', return_value=config_req), + mock.patch.object(self.store.process, 'send_request_get_response', + return_value=mock.Mock(success=False)) + ) as (request_patch, response_patch, update_patch, build_patch, send_patch): + self.store.load_config(new_config) + build_patch.assert_called_once_with(msg_enums.REQUEST_CONFIG, '', new_config) + send_patch.assert_called_once_with(config_req) + assert not update_patch.called + assert not request_patch.called + assert not response_patch.called diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index 805094565..7374280e3 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -190,12 +190,9 @@ def test_send_request_shutdown_not_shutdown_but_dead(self): assert not terminate_patch.called def test_update_config(self): - request = mock.Mock() new_config = mock.Mock() - with mock.patch.object(self.process, 'send_request') as send_patch: - self.process.update_config(new_config, request) - send_patch.assert_called_once_with(request) - assert_equal(self.process.config, new_config) + self.process.update_config(new_config) + assert_equal(self.process.config, new_config) def test_poll_for_response_has_response_makes_orphaned(self): self.process.orphaned_responses = {} diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py index 6d21b576c..194c871ae 100644 --- a/tests/serialize/runstate/tronstore/tronstore_test.py +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -90,7 +90,7 @@ def test_shutdown_request(self): self.response_factory.build.assert_called_once_with(True, fake_id, '') self.pipe.send_bytes.assert_called_once_with(self.response_factory.build().serialized) - def test_config_request(self): + def test_config_request_success(self): fake_id = 77 fake_shutdown_id = 88 self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), @@ -102,9 +102,35 @@ def test_config_request(self): assert_equal(self.store_class.cleanup.call_count, 2) self.store_class.cleanup.assert_any_call() assert_equal(self.parse_patch.call_count, 2) + self.queue.empty.assert_called_once_with() + self.response_factory.build.assert_any_call(True, fake_id, '') + self.pipe.send_bytes.assert_any_call(self.response_factory.build().serialized) self.request_factory.update_method.assert_called_once_with(self.trans_method) self.response_factory.update_method.assert_called_once_with(self.trans_method) + def test_config_request_exception(self): + fake_id = 77 + fake_shutdown_id = 88 + some_fake_store = mock.Mock() + some_fake_trans = mock.Mock() + self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), + mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] + item_iter = iter([(self.store_class, self.trans_method), 'breakit', + (some_fake_store, some_fake_trans)]) + self.parse_patch.configure_mock(side_effect=item_iter) + assert_raises(SystemError, tronstore.main, self.config, self.pipe) + + self.assert_main_startup() + assert tronstore.is_shutdown + self.store_class.cleanup.assert_called_once_with() + some_fake_store.cleanup.assert_called_once_with() + assert_equal(self.parse_patch.call_count, 3) + self.queue.empty.assert_called_once_with() + self.response_factory.build.assert_any_call(False, fake_id, '') + self.pipe.send_bytes.assert_any_call(self.response_factory.build().serialized) + self.request_factory.update_method.assert_called_once_with(some_fake_trans) + self.response_factory.update_method.assert_called_once_with(some_fake_trans) + def test_save_request(self): fake_id = 77 fake_shutdown_id = 88 diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 09cbeff4c..225c6157f 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -202,7 +202,7 @@ def _save_from_buffer(self): def update_from_config(self, new_state_config): self._save_from_buffer() - self._impl.load_config(new_state_config) + return self._impl.load_config(new_state_config) def cleanup(self): self._save_from_buffer() @@ -257,8 +257,8 @@ def update_from_config(self, state_config): if self.state_manager is NullStateManager: self.state_manager = PersistenceManagerFactory.from_config(state_config) - else: - self.state_manager.update_from_config(state_config) + elif not self.state_manager.update_from_config(state_config): + return False self.config = state_config return True diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index 2784938f3..dd97ed0a7 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -69,9 +69,14 @@ def load_config(self, new_config): """Reconfigure the storing mechanism to use a new configuration by shutting down and restarting tronstore.""" config_req = self.request_factory.build(msg_enums.REQUEST_CONFIG, '', new_config) - self.process.update_config(new_config, config_req) - self.request_factory.update_method(new_config.transport_method) - self.response_factory.update_method(new_config.transport_method) + response = self.process.send_request_get_response(config_req) + if response.success: + self.process.update_config(new_config, config_req) + self.request_factory.update_method(new_config.transport_method) + self.response_factory.update_method(new_config.transport_method) + return True + else: + return False def __repr__(self): return "ParallelStore" diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index bb52aa406..7d036adf9 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -113,8 +113,8 @@ def send_request_shutdown(self, request): for tronstore to send a response, after which it kills both pipes and the process itself. - Calling this prevents ANY further requests being made to tronstore, as - the process will be terminated. + Calling this prevents ANY further requests from being made to tronstore + as the process will be killed. """ if self.is_shutdown or not self.process.is_alive(): self.pipe.close() @@ -133,9 +133,11 @@ def send_request_shutdown(self, request): # to the process, which unfortunately is registered to do nothing # (as the process depends on trond to shut itself down, and shuts # itself down if trond is dead anyway.) - # We want a hard kill anyway. + # We want a hard kill regardless. os.kill(self.process.pid, signal.SIGKILL) - def update_config(self, new_config, config_request): - self.send_request(config_request) + def update_config(self, new_config): + """Update the configuration. Needed to make sure that tronstore + is restarted with the correct configuration upon exiting + prematurely.""" self.config = new_config diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index d4e5a4c0d..3e79c1f08 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -159,12 +159,22 @@ def main(config, pipe): shutdown_req_id = request.id elif request.req_type == msg_enums.REQUEST_CONFIG: - while len(running_threads) != 0: + while len(running_threads) != 0 or not thread_queue.empty(): time.sleep(0.5) store_class.cleanup() - store_class, transport_method = parse_config(request.data) - request_factory.update_method(transport_method) - response_factory.update_method(transport_method) + try: + # Try to set up with the new configuration + store_class, transport_method = parse_config(request.data) + pipe.send_bytes(response_factory.build(True, request.id, '').serialized) + request_factory.update_method(transport_method) + response_factory.update_method(transport_method) + config = request.data + except: + # Failed, go back to what we had + store_class, transport_method = parse_config(config) + request_factory.update_method(transport_method) + response_factory.update_method(transport_method) + pipe.send_bytes(response_factory.build(False, request.id, '').serialized) else: request_thread = Thread(target=handle_request, @@ -178,7 +188,7 @@ def main(config, pipe): request_thread.daemon = True thread_queue.put(request_thread) elif is_shutdown: - # We have to wait for all requests to clean up first. + # We have to wait for all threads to clean up first. while len(running_threads) != 0 or thread_pool.is_alive(): time.sleep(0.5) store_class.cleanup() From 57cb0f389773a49eb60fe27516aadf8a9d11f7ca Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 23 Jul 2013 17:37:17 -0700 Subject: [PATCH 18/48] some cleanup --- tron/config/config_parse.py | 2 +- tron/serialize/runstate/statemanager.py | 1 - tron/serialize/runstate/tronstore/messages.py | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index f664c1d77..3241b6be4 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -438,7 +438,7 @@ def validate_jobs_and_services(config, config_context): config_utils.unique_names(fmt_string, config['jobs'], config['services']) -DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', 'pickle', 1, 'json', 'pickle') +DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', None, 1, 'json', 'pickle') DEFAULT_NODE = ValidateNode().do_shortcut('localhost') diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 225c6157f..11098001b 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -25,7 +25,6 @@ class PersistenceStoreError(ValueError): class PersistenceManagerFactory(object): """Create a PersistentStateManager.""" - # TODO: Remove this class, it's somewhat pointless now @classmethod def from_config(cls, persistence_config): diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index c8524331c..c049f01e5 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -1,4 +1,7 @@ -from tron.serialize.runstate.tronstore.transport import JSONTransport, cPickleTransport, MsgPackTransport, YamlTransport +from tron.serialize.runstate.tronstore.transport import JSONTransport +from tron.serialize.runstate.tronstore.transport import cPickleTransport +from tron.serialize.runstate.tronstore.transport import MsgPackTransport +from tron.serialize.runstate.tronstore.transport import YamlTransport transport_class_map = { 'json': JSONTransport, From c2a85cccaa6c6616d880553b0d39ea419b401cd0 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 24 Jul 2013 17:35:12 -0700 Subject: [PATCH 19/48] changed tronstore to use a nullstore object by default, updated classes accordingly to match other tron patterns --- .../migrate_state_from_0.6.1_to_0.6.2.py | 17 ++-- tron/config/config_parse.py | 7 +- tron/config/schema.py | 3 +- tron/serialize/runstate/statemanager.py | 78 ++++++++++++------- tron/serialize/runstate/tronstore/messages.py | 76 +++++++++--------- .../runstate/tronstore/parallelstore.py | 14 ++-- tron/serialize/runstate/tronstore/process.py | 25 +++--- .../tronstore/{transport.py => serialize.py} | 30 ++++--- tron/serialize/runstate/tronstore/store.py | 22 +++++- .../serialize/runstate/tronstore/tronstore.py | 30 +++---- 10 files changed, 179 insertions(+), 123 deletions(-) rename tron/serialize/runstate/tronstore/{transport.py => serialize.py} (66%) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index d2eb9c5f4..6b1df5b37 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -34,11 +34,6 @@ configuration file. Options for str are sql, mongo, yaml, and shelve. - -t str Set a new method for transporting the new state objects to - tronstore. Defaults to whatever was set to transport_method in the - Tron configuration file, or pickle if it isn't set. - Options for str are pickle, yaml, msgpack, and json. - -d str Set a new method for storing state data within an SQL database. Defaults to whatever was set to db_store_method in the Tron configuration file, or json if it isn't set. Only used if @@ -55,7 +50,7 @@ from tron.commands import cmd_utils from tron.config import ConfigError -from tron.config.schema import StatePersistenceTypes, StateTransportTypes +from tron.config.schema import StatePersistenceTypes from tron.config.manager import ConfigManager from tron.serialize import runstate from tron.serialize.runstate.shelvestore import ShelveStateStore @@ -74,9 +69,9 @@ def parse_options(): parser.add_option("-m", type="string", help="Set new state storing mechanism (store_type)", dest="store_method", default=None) - parser.add_option("-t", type="string", - help="Set new transport method", - dest="transport_method", default=None) + # parser.add_option("-t", type="string", + # help="Set new transport method", + # dest="transport_method", default=None) parser.add_option("-d", type="string", help="Set new SQL db serialization method (db_store_method)", dest="db_store_method", default=None) @@ -118,8 +113,8 @@ def compile_new_info(options, state_info, new_file): if options.store_method: new_state_info = new_state_info._replace(store_method=options.store_method) - if options.transport_method: - new_state_info = new_state_info._replace(transport_method=options.transport_method) + # if options.transport_method: + # new_state_info = new_state_info._replace(transport_method=options.transport_method) if options.db_store_method: new_state_info = new_state_info._replace(db_store_method=options.db_store_method) diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 3241b6be4..477083826 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -399,19 +399,16 @@ class ValidateStatePersistence(Validator): # defaulting people into using a Turing Complete serialization method # for SQL storing. 'db_store_method': 'json', - 'transport_method': 'pickle', } validators = { 'name': valid_string, 'store_type': config_utils.build_enum_validator( schema.StatePersistenceTypes), - 'transport_method': config_utils.build_enum_validator( - schema.StateTransportTypes), 'connection_details': valid_string, 'buffer_size': valid_int, 'db_store_method': config_utils.build_enum_validator( - schema.StateTransportTypes), + schema.StateSerializationTypes), } def post_validation(self, config, config_context): @@ -438,7 +435,7 @@ def validate_jobs_and_services(config, config_context): config_utils.unique_names(fmt_string, config['jobs'], config['services']) -DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', None, 1, 'json', 'pickle') +DEFAULT_STATE_PERSISTENCE = ConfigState('tron_state', 'shelve', None, 1, 'json') DEFAULT_NODE = ValidateNode().do_shortcut('localhost') diff --git a/tron/config/schema.py b/tron/config/schema.py index ccaeea50d..94c1f50bd 100644 --- a/tron/config/schema.py +++ b/tron/config/schema.py @@ -94,7 +94,6 @@ def config_object_factory(name, required=None, optional=None): 'connection_details', 'buffer_size', 'db_store_method', - 'transport_method', ]) @@ -153,7 +152,7 @@ def config_object_factory(name, required=None, optional=None): StatePersistenceTypes = Enum.create('shelve', 'sql', 'mongo', 'yaml') -StateTransportTypes = Enum.create('json', 'pickle', 'msgpack', 'yaml') +StateSerializationTypes = Enum.create('json', 'pickle', 'msgpack', 'yaml') ActionRunnerTypes = Enum.create('none', 'subprocess') diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 11098001b..2b08d85dc 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -23,28 +23,28 @@ class PersistenceStoreError(ValueError): """Raised if the store can not be created or fails a read or write.""" -class PersistenceManagerFactory(object): - """Create a PersistentStateManager.""" +# class PersistenceManagerFactory(object): +# """Create a PersistentStateManager.""" - @classmethod - def from_config(cls, persistence_config): - store_type = persistence_config.store_type - transport_method = persistence_config.transport_method - db_store_method = persistence_config.db_store_method - buffer_size = persistence_config.buffer_size +# @classmethod +# def from_config(cls, persistence_config): +# store_type = persistence_config.store_type +# # transport_method = persistence_config.transport_method +# db_store_method = persistence_config.db_store_method +# buffer_size = persistence_config.buffer_size - if store_type not in schema.StatePersistenceTypes: - raise PersistenceStoreError("Unknown store type: %s" % store_type) +# if store_type not in schema.StatePersistenceTypes: +# raise PersistenceStoreError("Unknown store type: %s" % store_type) - if transport_method not in schema.StateTransportTypes: - raise PersistenceStoreError("Unknown transport type: %s" % transport_method) +# # if transport_method not in schema.StateTransportTypes: +# # raise PersistenceStoreError("Unknown transport type: %s" % transport_method) - if db_store_method not in schema.StateTransportTypes and store_type in ('sql', 'mongo'): - raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) +# if db_store_method not in schema.StateSerializationTypes and store_type in ('sql', 'mongo'): +# raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) - store = ParallelStore(persistence_config) - buffer = StateSaveBuffer(buffer_size) - return PersistentStateManager(store, buffer) +# store = ParallelStore(persistence_config) +# buffer = StateSaveBuffer(buffer_size) +# return PersistentStateManager(store, buffer) class StateMetadata(object): @@ -105,6 +105,18 @@ def __iter__(self): self.buffer.clear() +class NullSaveBuffer(object): + buffer_size = 0 + buffer = {} + counter = 0 + + def save(self, key, state_data): + return False + + def __iter__(self): + return iter([]) + + class PersistentStateManager(object): """Provides an interface to persist the state of Tron. @@ -131,10 +143,10 @@ def cleanup(self): """ # TODO: Rename things here, as ParallelStore is always used - def __init__(self, persistence_impl, buffer): + def __init__(self): self.enabled = True - self._buffer = buffer - self._impl = persistence_impl + self._buffer = NullSaveBuffer() + self._impl = ParallelStore() self.metadata_key = self._impl.build_key( runstate.MCP_STATE, StateMetadata.name) @@ -201,7 +213,11 @@ def _save_from_buffer(self): def update_from_config(self, new_state_config): self._save_from_buffer() - return self._impl.load_config(new_state_config) + if self._impl.load_config(new_state_config): + self._buffer = StateSaveBuffer(new_state_config.buffer_size) + return True + else: + return False def cleanup(self): self._save_from_buffer() @@ -247,19 +263,27 @@ class StateChangeWatcher(observer.Observer): """Observer of stateful objects.""" def __init__(self): - self.state_manager = NullStateManager + self.state_manager = PersistentStateManager() self.config = None def update_from_config(self, state_config): if self.config == state_config: return False - if self.state_manager is NullStateManager: - self.state_manager = PersistenceManagerFactory.from_config(state_config) - elif not self.state_manager.update_from_config(state_config): + if state_config.store_type not in schema.StatePersistenceTypes: + raise PersistenceStoreError("Unknown store type: %s" % store_type) + + if state_config.db_store_method not in schema.StateSerializationTypes \ + and store_type in ('sql', 'mongo'): + raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) + + # if self.state_manager is NullStateManager: + # self.state_manager = PersistenceManagerFactory.from_config(state_config) + if not self.state_manager.update_from_config(state_config): return False - self.config = state_config - return True + else: + self.config = state_config + return True def handler(self, observable, _event): """Handle a state change in an observable by saving its state.""" diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index c049f01e5..4fe4d69bc 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -1,14 +1,8 @@ -from tron.serialize.runstate.tronstore.transport import JSONTransport -from tron.serialize.runstate.tronstore.transport import cPickleTransport -from tron.serialize.runstate.tronstore.transport import MsgPackTransport -from tron.serialize.runstate.tronstore.transport import YamlTransport - -transport_class_map = { - 'json': JSONTransport, - 'pickle': cPickleTransport, - 'msgpack': MsgPackTransport, - 'yaml': YamlTransport -} +# from tron.serialize.runstate.tronstore.transport import JSONTransport +from tron.serialize.runstate.tronstore.serialize import cPickleSerializer +# from tron.serialize.runstate.tronstore.transport import MsgPackTransport +# from tron.serialize.runstate.tronstore.transport import YamlTransport + # a simple max integer to prevent ids from growing indefinitely MAX_MSG_ID = 2**32 - 1 @@ -16,8 +10,8 @@ class StoreRequestFactory(object): """A factory to generate requests by giving each a unique id.""" - def __init__(self, method): - self.serializer = transport_class_map[method] + def __init__(self): + self.serializer = cPickleSerializer self.id_counter = 1 def _increment_counter(self): @@ -34,9 +28,9 @@ def build(self, req_type, data_type, data): def rebuild(self, msg): return StoreRequest.from_message(self.serializer.deserialize(msg), self.serializer) - def update_method(self, new_method): - """Update the method used for message serialization.""" - self.serializer = transport_class_map[new_method] + # def update_method(self, new_method): + # """Update the method used for message serialization.""" + # self.serializer = transport_class_map[new_method] def get_method(self): return self.serializer @@ -49,8 +43,8 @@ class StoreResponseFactory(object): StoreResponse objects using that method. """ - def __init__(self, method): - self.serializer = transport_class_map[method] + def __init__(self): + self.serializer = cPickleSerializer def build(self, success, req_id, data): new_request = StoreResponse(req_id, success, data, self.serializer) @@ -59,9 +53,9 @@ def build(self, success, req_id, data): def rebuild(self, msg): return StoreResponse.from_message(self.serializer.deserialize(msg), self.serializer) - def update_method(self, new_method): - """Update the method used for message serialization.""" - self.serializer = transport_class_map[new_method] + # def update_method(self, new_method): + # """Update the method used for message serialization.""" + # self.serializer = transport_class_map[new_method] def get_method(self): return self.serializer @@ -83,25 +77,33 @@ def __init__(self, req_id, req_type, data_type, data, method): self.data = data self.data_type = data_type self.method = method - self.serialized = self.get_serialized() + # self.serialized = self.get_serialized() @classmethod def from_message(cls, msg_data, method): req_id, req_type, data_type, data = msg_data return cls(req_id, req_type, data_type, data, method) - def update_method(self, new_method): - """Update the method used for message serialization.""" - self.method = transport_class_map['new_method'] - self.serialized = self.get_serialized() - - def get_serialized(self): + @property + def serialized(self): return self.method.serialize(( self.id, self.req_type, self.data_type, self.data)) + # def update_method(self, new_method): + # """Update the method used for message serialization.""" + # self.method = transport_class_map['new_method'] + # self.serialized = self.get_serialized() + + # def get_serialized(self): + # return self.method.serialize(( + # self.id, + # self.req_type, + # self.data_type, + # self.data)) + class StoreResponse(object): """An object representing a response from tronstore. The response has three @@ -116,17 +118,21 @@ def __init__(self, req_id, success, data, method): self.success = success self.data = data self.method = method - self.serialized = self.get_serialized() + # self.serialized = self.get_serialized() @classmethod def from_message(cls, msg_data, method): req_id, success, data = msg_data return cls(req_id, success, data, method) - def update_method(self, new_method): - """Update the method used for message serialization.""" - self.method = transport_class_map['new_method'] - self.serialized = self.get_serialized() - - def get_serialized(self): + @property + def serialized(self): return self.method.serialize((self.id, self.success, self.data)) + + # def update_method(self, new_method): + # """Update the method used for message serialization.""" + # self.method = transport_class_map['new_method'] + # self.serialized = self.get_serialized() + + # def get_serialized(self): + # return self.method.serialize((self.id, self.success, self.data)) diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index dd97ed0a7..b65fd8704 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -4,7 +4,7 @@ import os from tron.serialize.runstate.tronstore.process import StoreProcessProtocol -from tron.serialize.runstate.tronstore.messages import StoreRequestFactory, StoreResponseFactory +from tron.serialize.runstate.tronstore.messages import StoreRequestFactory from tron.serialize.runstate.tronstore import msg_enums log = logging.getLogger(__name__) @@ -30,6 +30,7 @@ def __eq__(self, other): def __hash__(self): return hash(self.key) + class ParallelStore(object): """Persist state using a paralleled storing mechanism, tronstore. This uses the Twisted library to run the tronstore executable in a separate @@ -38,10 +39,9 @@ class ParallelStore(object): This class handles construction of all messages that need to be sent to tronstore based on requests given by the MCP.""" - def __init__(self, config): - self.request_factory = StoreRequestFactory(config.transport_method) - self.response_factory = StoreResponseFactory(config.transport_method) - self.process = StoreProcessProtocol(config, self.response_factory) + def __init__(self): + self.request_factory = StoreRequestFactory() + self.process = StoreProcessProtocol() def build_key(self, type, iden): return ParallelKey(type, iden) @@ -72,8 +72,8 @@ def load_config(self, new_config): response = self.process.send_request_get_response(config_req) if response.success: self.process.update_config(new_config, config_req) - self.request_factory.update_method(new_config.transport_method) - self.response_factory.update_method(new_config.transport_method) + # self.request_factory.update_method(new_config.transport_method) + # self.response_factory.update_method(new_config.transport_method) return True else: return False diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index 7d036adf9..e3f33d548 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -4,16 +4,20 @@ from multiprocessing import Process, Pipe from tron.serialize.runstate.tronstore import tronstore +from tron.serialize.runstate.tronstore.messages import StoreResponseFactory log = logging.getLogger(__name__) + class TronStoreError(Exception): """Raised whenever tronstore exits for an unknown reason.""" def __init__(self, code): self.code = code + def __str__(self): return repr(self.code) + class StoreProcessProtocol(object): """The class that actually communicates with tronstore. This is a subclass of the twisted ProcessProtocol class, which has a set of internals that can @@ -28,9 +32,9 @@ class StoreProcessProtocol(object): SHUTDOWN_TIMEOUT = 100.0 POLL_TIMEOUT = 10.0 - def __init__(self, config, response_factory): - self.config = config - self.response_factory = response_factory + def __init__(self): + self.config = None + self.response_factory = StoreResponseFactory() self.orphaned_responses = {} self.is_shutdown = False self._start_process() @@ -65,7 +69,6 @@ def send_request(self, request): self._verify_is_alive() self.pipe.send_bytes(request.serialized) - # self.transport.write(self.chunker.sign(request.serialized)) def _poll_for_response(self, id, timeout): """Polls for a response to the request with identifier id. Throws @@ -78,9 +81,9 @@ def _poll_for_response(self, id, timeout): be fine. """ if id in self.orphaned_responses: - response = self.orphaned_responses[id] - del self.orphaned_responses[id] - return response + # response = self.orphaned_responses[id] + # del self.orphaned_responses[id] + return self.orphaned_responses.pop(id) while self.pipe.poll(timeout): response = self.response_factory.rebuild(self.pipe.recv_bytes()) @@ -103,7 +106,8 @@ def send_request_get_response(self, request): self.pipe.send_bytes(request.serialized) response = self._poll_for_response(request.id, self.POLL_TIMEOUT) if not response: - log.warn("tronstore took longer than %d seconds to respond to a request, and it was dropped." % self.POLL_TIMEOUT) + log.warn(("tronstore took longer than %d seconds to respond to a" + "request, and it was dropped.") % self.POLL_TIMEOUT) return self.response_factory.build(False, request.id, '') else: return response @@ -134,7 +138,10 @@ def send_request_shutdown(self, request): # (as the process depends on trond to shut itself down, and shuts # itself down if trond is dead anyway.) # We want a hard kill regardless. - os.kill(self.process.pid, signal.SIGKILL) + try: + os.kill(self.process.pid, signal.SIGKILL) + except: + pass def update_config(self, new_config): """Update the configuration. Needed to make sure that tronstore diff --git a/tron/serialize/runstate/tronstore/transport.py b/tron/serialize/runstate/tronstore/serialize.py similarity index 66% rename from tron/serialize/runstate/tronstore/transport.py rename to tron/serialize/runstate/tronstore/serialize.py index e696dcbdf..de38ae293 100644 --- a/tron/serialize/runstate/tronstore/transport.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -1,4 +1,4 @@ -"""Message transport modules for tronstore. This allows for simple writing +"""Message serialization modules for tronstore. This allows for simple writing of stdin/out with strings that can then be put back into tuples of data for rebuilding messages. @@ -21,8 +21,8 @@ no_yaml = True -class TransportModuleError(Exception): - """Raised if a transport module is used without it being installed.""" +class SerializerModuleError(Exception): + """Raised if a serialization module is used without it being installed.""" def __init__(self, code): self.code = code @@ -30,7 +30,7 @@ def __str__(self): return repr(self.code) -class JSONTransport(object): +class JSONSerializer(object): @classmethod def serialize(cls, data): return json.dumps(data, tuple_as_array=False) @@ -40,7 +40,7 @@ def deserialize(cls, data_str): return json.loads(data_str) -class cPickleTransport(object): +class cPickleSerializer(object): @classmethod def serialize(cls, data): return pickle.dumps(data) @@ -50,29 +50,37 @@ def deserialize(cls, data_str): return pickle.loads(data_str) -class MsgPackTransport(object): +class MsgPackSerializer(object): @classmethod def serialize(cls, data): if no_msgpack: - raise TransportModuleError('MessagePack not installed.') + raise SerializerModuleError('MessagePack not installed.') return msgpack.packb(data) @classmethod def deserialize(cls, data_str): if no_msgpack: - raise TransportModuleError('MessagePack not installed.') + raise SerializerModuleError('MessagePack not installed.') return msgpack.unpackb(data_str, use_list=False) -class YamlTransport(object): +class YamlSerializer(object): @classmethod def serialize(cls, data): if no_yaml: - raise TransportModuleError('PyYaml not installed.') + raise SerializerModuleError('PyYaml not installed.') return yaml.dump(data) @classmethod def deserialize(cls, data_str): if no_yaml: - raise TransportModuleError('PyYaml not installed.') + raise SerializerModuleError('PyYaml not installed.') return yaml.load(data_str) + + +serialize_class_map = { + 'json': JSONSerializer, + 'pickle': cPickleSerializer, + 'msgpack': MsgPackSerializer, + 'yaml': YamlSerializer +} diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 4229ce1fd..cb4e922f3 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -3,11 +3,26 @@ import os from contextlib import contextmanager -from tron.serialize.runstate.tronstore.messages import transport_class_map +from tron.serialize.runstate.tronstore.transport import serialize_class_map from tron.serialize import runstate from tron.config.config_utils import MAX_IDENTIFIER_LENGTH +class NullStore(object): + + def save(self, key, state_data, data_type): + return False + + def restore(self, key, data_type): + return (False, None) + + def cleanup(self): + pass + + def __repr__(self): + return "NullStateStore" + + class ShelveStore(object): """Store state using python's built-in shelve module.""" @@ -170,6 +185,7 @@ def __repr__(self): class YamlStore(object): + # TODO: Deprecate this, it's bad """Store state in a local YAML file. WARNING: Using this is NOT recommended, even moreso than the previous @@ -228,5 +244,5 @@ def __repr__(self): def build_store(name, store_type, connection_details, db_store_method): - trans_class = transport_class_map[db_store_method] if db_store_method != "None" else None - return store_class_map[store_type](name, connection_details, trans_class) + serial_class = serialize_class_map[db_store_method] if db_store_method != "None" else None + return store_class_map[store_type](name, connection_details, serial_class) diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 3e79c1f08..9c02e028c 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -36,13 +36,16 @@ def _discard_signal(signum, frame): def parse_config(config): """Parse the configuration file and set up the store class.""" + if not config: + return store.NullStore() + name = config.name - transport_method = config.transport_method + # transport_method = config.transport_method store_type = config.store_type connection_details = config.connection_details db_store_method = config.db_store_method - return (store.build_store(name, store_type, connection_details, db_store_method), transport_method) + return store.build_store(name, store_type, connection_details, db_store_method) #, transport_method) def get_all_from_pipe(pipe): """Gets all of the requests from the pipe, returning an array of serialized @@ -131,10 +134,10 @@ def main(config, pipe): is_shutdown = False shutdown_req_id = None - store_class, transport_method = parse_config(config) + store_class = parse_config(config) - request_factory = StoreRequestFactory(transport_method) - response_factory = StoreResponseFactory(transport_method) + request_factory = StoreRequestFactory() + response_factory = StoreResponseFactory() save_lock = Lock() restore_lock = Lock() @@ -164,16 +167,16 @@ def main(config, pipe): store_class.cleanup() try: # Try to set up with the new configuration - store_class, transport_method = parse_config(request.data) + store_class = parse_config(request.data) pipe.send_bytes(response_factory.build(True, request.id, '').serialized) - request_factory.update_method(transport_method) - response_factory.update_method(transport_method) + # request_factory.update_method(transport_method) + # response_factory.update_method(transport_method) config = request.data except: # Failed, go back to what we had - store_class, transport_method = parse_config(config) - request_factory.update_method(transport_method) - response_factory.update_method(transport_method) + store_class = parse_config(config) + # request_factory.update_method(transport_method) + # response_factory.update_method(transport_method) pipe.send_bytes(response_factory.build(False, request.id, '').serialized) else: @@ -185,7 +188,7 @@ def main(config, pipe): response_factory, save_lock, restore_lock)) - request_thread.daemon = True + request_thread.daemon = False thread_queue.put(request_thread) elif is_shutdown: # We have to wait for all threads to clean up first. @@ -194,6 +197,7 @@ def main(config, pipe): store_class.cleanup() if shutdown_req_id: pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) + # TODO: Do we need a forceful kill? Can we just return here? os._exit(0) else: # Did tron die? @@ -204,4 +208,4 @@ def main(config, pipe): except IOError, e: # Error #4 is a system interrupt, caused by ^C if e.errno != 4: - raise e + raise From 23659f03aeb3540f0645521471eacd3942ee4970 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 24 Jul 2013 17:42:40 -0700 Subject: [PATCH 20/48] renamed rebuild function in request/response factories to from_msg --- .../runstate/tronstore/process_test.py | 20 +++++++++---------- .../runstate/tronstore/tronstore_test.py | 2 +- tron/serialize/runstate/tronstore/messages.py | 4 ++-- tron/serialize/runstate/tronstore/process.py | 2 +- .../serialize/runstate/tronstore/tronstore.py | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index 7374280e3..d9cb8a4e9 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -217,16 +217,16 @@ def get_fake_response(fake_response_serial): with contextlib.nested( mock.patch.object(self.process.pipe, 'poll', return_value=True), mock.patch.object(self.process.pipe, 'recv_bytes', side_effect=recv_change_response), - mock.patch.object(self.process.response_factory, 'rebuild', side_effect=get_fake_response) - ) as (poll_patch, recv_patch, rebuild_patch): + mock.patch.object(self.process.response_factory, 'from_msg', side_effect=get_fake_response) + ) as (poll_patch, recv_patch, from_msg_patch): assert_equal(self.process._poll_for_response(fake_id, fake_timeout), fake_response_matching) assert_equal(self.process.orphaned_responses, {fake_id_other: fake_response_other}) poll_patch.assert_called_with(fake_timeout) assert_equal(poll_patch.call_count, 2) recv_patch.assert_called_with() assert_equal(recv_patch.call_count, 2) - rebuild_patch.assert_called_with('second') - assert_equal(rebuild_patch.call_count, 2) + from_msg_patch.assert_called_with('second') + assert_equal(from_msg_patch.call_count, 2) def test_poll_for_response_has_orphaned(self): fake_id = 77 @@ -236,13 +236,13 @@ def test_poll_for_response_has_orphaned(self): with contextlib.nested( mock.patch.object(self.process.pipe, 'poll', return_value=True), mock.patch.object(self.process.pipe, 'recv_bytes'), - mock.patch.object(self.process.response_factory, 'rebuild') - ) as (poll_patch, recv_patch, rebuild_patch): + mock.patch.object(self.process.response_factory, 'from_msg') + ) as (poll_patch, recv_patch, from_msg_patch): assert_equal(self.process._poll_for_response(fake_id, fake_timeout), fake_response) assert_equal(self.process.orphaned_responses, {}) assert not poll_patch.called assert not recv_patch.called - assert not rebuild_patch.called + assert not from_msg_patch.called def test_poll_for_response_no_response(self): fake_id = 77 @@ -251,10 +251,10 @@ def test_poll_for_response_no_response(self): with contextlib.nested( mock.patch.object(self.process.pipe, 'poll', return_value=False), mock.patch.object(self.process.pipe, 'recv_bytes'), - mock.patch.object(self.process.response_factory, 'rebuild') - ) as (poll_patch, recv_patch, rebuild_patch): + mock.patch.object(self.process.response_factory, 'from_msg') + ) as (poll_patch, recv_patch, from_msg_patch): assert_equal(self.process._poll_for_response(fake_id, fake_timeout), None) assert_equal(self.process.orphaned_responses, {}) poll_patch.assert_called_once_with(fake_timeout) assert not recv_patch.called - assert not rebuild_patch.called + assert not from_msg_patch.called diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py index 194c871ae..6363d6049 100644 --- a/tests/serialize/runstate/tronstore/tronstore_test.py +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -26,7 +26,7 @@ def poll_patch(timeout): def echo_single_request(request): return request - self.request_factory.rebuild = echo_single_request + self.request_factory.from_msg = echo_single_request def echo_requests(not_used): return self.requests diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 4fe4d69bc..49257f23f 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -25,7 +25,7 @@ def build(self, req_type, data_type, data): self.id_counter = self._increment_counter() return new_request - def rebuild(self, msg): + def from_msg(self, msg): return StoreRequest.from_message(self.serializer.deserialize(msg), self.serializer) # def update_method(self, new_method): @@ -50,7 +50,7 @@ def build(self, success, req_id, data): new_request = StoreResponse(req_id, success, data, self.serializer) return new_request - def rebuild(self, msg): + def from_msg(self, msg): return StoreResponse.from_message(self.serializer.deserialize(msg), self.serializer) # def update_method(self, new_method): diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index e3f33d548..a5faa4bd1 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -86,7 +86,7 @@ def _poll_for_response(self, id, timeout): return self.orphaned_responses.pop(id) while self.pipe.poll(timeout): - response = self.response_factory.rebuild(self.pipe.recv_bytes()) + response = self.response_factory.from_msg(self.pipe.recv_bytes()) if response.id == id: return response else: diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 9c02e028c..f2d2eae17 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -155,7 +155,7 @@ def main(config, pipe): try: if pipe.poll(POLL_TIMEOUT): requests = get_all_from_pipe(pipe) - requests = map(request_factory.rebuild, requests) + requests = map(request_factory.from_msg, requests) for request in requests: if request.req_type == msg_enums.REQUEST_SHUTDOWN: is_shutdown = True From 5061dbadb33e797e5f1bea753f78fd6b9c46b3c2 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 29 Jul 2013 15:40:52 -0700 Subject: [PATCH 21/48] refactor of tronstore.py into classes --- tron/core/job.py | 1 + .../runstate/tronstore/parallelstore.py | 2 +- tron/serialize/runstate/tronstore/process.py | 5 +- tron/serialize/runstate/tronstore/store.py | 39 +- .../serialize/runstate/tronstore/tronstore.py | 399 ++++++++++-------- 5 files changed, 277 insertions(+), 169 deletions(-) diff --git a/tron/core/job.py b/tron/core/job.py index d5a3f62d7..195416e33 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -441,6 +441,7 @@ def restore_state(self, state_data): self.watcher.watch(run) self.job_state.restore_state(job_state_data) self.job_scheduler.restore_state() + self.job_state.set_run_ids(self.job_runs.get_run_numbers()) # consistency self.event.ok('restored') def update_from_job(self, job): diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index b65fd8704..ebddbe8f9 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -71,7 +71,7 @@ def load_config(self, new_config): config_req = self.request_factory.build(msg_enums.REQUEST_CONFIG, '', new_config) response = self.process.send_request_get_response(config_req) if response.success: - self.process.update_config(new_config, config_req) + self.process.update_config(new_config) # self.request_factory.update_method(new_config.transport_method) # self.response_factory.update_method(new_config.transport_method) return True diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index a5faa4bd1..a082c7aa2 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -65,6 +65,7 @@ def send_request(self, request): waiting for tronstore's response. """ if self.is_shutdown: + log.warn('attempted to send a request of type %s while shut down!' % request.req_type) return self._verify_is_alive() @@ -100,6 +101,7 @@ def send_request_get_response(self, request): """ if self.is_shutdown: + log.warn('attempted to send a request of type %s while shut down!' % request.req_type) return self.response_factory.build(False, request.id, '') self._verify_is_alive() @@ -137,7 +139,8 @@ def send_request_shutdown(self, request): # to the process, which unfortunately is registered to do nothing # (as the process depends on trond to shut itself down, and shuts # itself down if trond is dead anyway.) - # We want a hard kill regardless. + # We want a hard kill regardless. The only way we should get to + # this code is if tronstore is about to call os._exit(0) itself. try: os.kill(self.process.pid, signal.SIGKILL) except: diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index cb4e922f3..3e2475fce 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -2,8 +2,9 @@ import urlparse import os from contextlib import contextmanager +from threading import Lock -from tron.serialize.runstate.tronstore.transport import serialize_class_map +from tron.serialize.runstate.tronstore.serialize import serialize_class_map from tron.serialize import runstate from tron.config.config_utils import MAX_IDENTIFIER_LENGTH @@ -246,3 +247,39 @@ def __repr__(self): def build_store(name, store_type, connection_details, db_store_method): serial_class = serialize_class_map[db_store_method] if db_store_method != "None" else None return store_class_map[store_type](name, connection_details, serial_class) + + +class SyncStore(object): + """A store object that synchronizes all save/restore operations on the + store implementation, as we have no idea what could happen due to its + modular nature. + """ + + def __init__(self, config): + """Parse the configuration file and set up the store class.""" + self.lock = Lock() + if not config: + self.store = NullStore() + + else: + name = config.name + store_type = config.store_type + connection_details = config.connection_details + db_store_method = config.db_store_method + + self.store = build_store(name, store_type, connection_details, + db_store_method) + + def save(self, *args, **kwargs): + with self.lock: + return self.store.save(*args, **kwargs) + + def restore(self, *args, **kwargs): + with self.lock: + return self.store.restore(*args, **kwargs) + + def cleanup(self): + self.store.cleanup() + + def __repr__(self): + return "SyncStore('%s')" % self.store.__repr__() diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index f2d2eae17..ce3334e8f 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -10,8 +10,8 @@ The process intercepts the two shutdown signals (SIGINT and SIGTERM) in order to prevent the process from exiting early when trond wants to do some final -shutdown things (realistically, it should be handling all shutdown operations -as this is a child process.) +shutdown things (realistically, trond should be handling all shutdown +operations, as this is a child process.) """ import time import signal @@ -23,189 +23,256 @@ from tron.serialize.runstate.tronstore import store from tron.serialize.runstate.tronstore import msg_enums -# this can be rather long- it's only real use it to clean up tronstore -# in case it's zombied... however, it should be SHORTER than -# SHUTDOWN_TIMEOUT in process.py -POLL_TIMEOUT = 2.0 -POOL_SIZE = 35 - def _discard_signal(signum, frame): pass -def parse_config(config): - """Parse the configuration file and set up the store class.""" - if not config: - return store.NullStore() - - name = config.name - # transport_method = config.transport_method - store_type = config.store_type - connection_details = config.connection_details - db_store_method = config.db_store_method +def _register_null_handlers(): + signal.signal(signal.SIGINT, _discard_signal) + signal.signal(signal.SIGHUP, _discard_signal) + signal.signal(signal.SIGTERM, _discard_signal) - return store.build_store(name, store_type, connection_details, db_store_method) #, transport_method) -def get_all_from_pipe(pipe): - """Gets all of the requests from the pipe, returning an array of serialized - requests (they still need to be decoded). +def handle_requests(request_queue, resp_factory, pipe, store_class, do_work): + """Handle requests by acting on store_class with the appropriate action. + Requests are taken from request_queue until do_work.val (it should be a + PoolBool) is False. + This is run in a separate thread. """ - requests = [] - while pipe.poll(): - requests.append(pipe.recv_bytes()) - return requests - -def handle_request(request, store_class, pipe, factory, save_lock, restore_lock): - """Handle a request by acting on store_class with the appropriate action. - - This is run in a separate thread. As such, there's two mutexes here- one - for the save requests, and one for the restore requests.""" - - if request.req_type == msg_enums.REQUEST_SAVE: - with save_lock: - success = store_class.save(request.data[0], request.data[1], request.data_type) - pipe.send_bytes(factory.build(success, request.id, '').serialized) - - elif request.req_type == msg_enums.REQUEST_RESTORE: - with restore_lock: - success, data = store_class.restore(request.data, request.data_type) - pipe.send_bytes(factory.build(success, request.id, data).serialized) - - else: - pipe.send_bytes(factory.build(False, request.id, '').serialized) - - -def _remove_finished_threads(running_threads): - """A small helper function to clean out the running_threads array. - Doesn't actually create a new instance of a list; it modifies - the existing list as a side effect, and returns the number - of running threads that it cleaned up.""" - counter = 0 - for i in range(len(running_threads) - 1, -1, -1): - if not running_threads[i].is_alive(): - running_threads.pop(i) - counter += 1 - return counter - - -def thread_starter(queue, running_threads): - """A method to start threads that have been queued up in queue. Also takes - a reference to a list (running_threads) that this function will store any - threads it has started in, so the main method knows if there's still - currently executing requests. - - Keep in mind because running_threads is a reference to a single instance - of a list object, it CANNOT be reassigned to another instance in order to - allow the main thread to know what's running. As such, all operations on - running_threads must be method calls to modify the list instance given - to this thread.""" - global is_shutdown, POOL_SIZE - - pool_counter = POOL_SIZE - - while not is_shutdown or not queue.empty(): - pool_counter += _remove_finished_threads(running_threads) - - if pool_counter <= 0: - time.sleep(0.5) - continue + # This should probably be lower rather than higher + WORK_TIMEOUT = 1.0 + + while do_work.val or not request_queue.empty(): try: - thread = queue.get(timeout=0.5) - thread.start() - pool_counter -= 1 - running_threads.append(thread) + request = request_queue.get(block=True, timeout=WORK_TIMEOUT) + + if request.req_type == msg_enums.REQUEST_SAVE: + store_class.save(request.data[0], request.data[1], request.data_type) + # pipe.send_bytes(resp_factory.build(success, request.id, '').serialized) + + elif request.req_type == msg_enums.REQUEST_RESTORE: + success, data = store_class.restore(request.data, request.data_type) + pipe.send_bytes(resp_factory.build(success, request.id, data).serialized) + + else: + pipe.send_bytes(resp_factory.build(False, request.id, '').serialized) + except Empty: continue - while len(running_threads) != 0: - _remove_finished_threads(running_threads) +class SyncPipe(object): + """An object to handle synchronization over pipe operations. In particular, + the send and recv functions should have mutexes as they are subject to + race conditions. + """ + + def __init__(self, pipe): + self.lock = Lock() + self.pipe = pipe -def main(config, pipe): - """The main run loop for tronstore. This loop sets up everything - based on the configuration tronstore got, and then simply - waits for requests to handle from pipe. It spawns threads for - save and restore requests, which will send responses back over - the pipe once completed.""" - global is_shutdown, POLL_TIMEOUT + # None is actually a valid timeout (blocks forever), so we need to use + # something different for checking for a non-supplied kwarg + def poll(self, *args, **kwargs): + return self.pipe.poll(*args, **kwargs) - is_shutdown = False - shutdown_req_id = None + def send_bytes(self, *args, **kwargs): + with self.lock: + return self.pipe.send_bytes(*args, **kwargs) - store_class = parse_config(config) + def recv_bytes(self, *args, **kwargs): + with self.lock: + return self.pipe.recv_bytes(*args, **kwargs) - request_factory = StoreRequestFactory() - response_factory = StoreResponseFactory() - save_lock = Lock() - restore_lock = Lock() - signal.signal(signal.SIGINT, _discard_signal) - signal.signal(signal.SIGHUP, _discard_signal) - signal.signal(signal.SIGTERM, _discard_signal) +class PoolBool(object): + """The PoolBool(TM) is a mutable boolean wrapper used for signaling.""" - running_threads = [] - thread_queue = Queue() - thread_pool = Thread(target=thread_starter, args=(thread_queue, running_threads)) - thread_pool.daemon = False - thread_pool.start() + def __init__(self, value=True): + if not value in (True, False): + raise TypeError('expected boolean, got %r' % value) + self._val = value - while True: + @property + def value(self): + return self._val + val = value + + def set(self, value): + if not value in (True, False): + raise TypeError('expected boolean, got %r' % value) + self._val = value + + +class TronstorePool(object): + """A thread pool with POOL_SIZE workers for handling requests. Enqueues + save and restore requests into a queue that is then consumed by the + workers, which send an appropriate response. + """ + + POOL_SIZE = 35 + + def __init__(self, resp_fact, pipe, store): + """Initialize the thread pool. Please make a new pool if any of the + objects passed to __init__ change. + """ + self.request_queue = Queue() + self.response_factory = resp_fact + self.pipe = pipe + self.store_class = store + self.keep_working = PoolBool(True) + self.thread_pool = [Thread(target=handle_requests, + args=( + self.request_queue, + self.response_factory, + self.pipe, + self.store_class, + self.keep_working + )) for i in range(self.POOL_SIZE)] + + def start(self): + """Start the thread pool.""" + self.keep_working.set(True) + for thread in self.thread_pool: + thread.daemon = False + thread.start() + + def stop(self): + """Stop the thread pool.""" + self.keep_working.set(False) + while self.has_work() \ + or any([thread.is_alive() for thread in self.thread_pool]): + time.sleep(0.5) + + def enqueue_work(self, work): + """Enqueue a request for the workers to consume and process.""" + self.request_queue.put(work) + + def has_work(self): + """Returns whether there is still work to be consumed by workers.""" + return not self.request_queue.empty() + + +class TronstoreMain(object): + """The main Tronstore class. Initializes a bunch of stuff and then has a + main_loop function that loops and handles requests from trond. + """ + + # this can be rather long- it's only real use it to clean up tronstore + # in case it's zombied... however, it should be SHORTER than + # SHUTDOWN_TIMEOUT in process.py. in addition, making this longer + # can cause trond to take longer to fully shutdown. + POLL_TIMEOUT = 2.0 + + def __init__(self, config, pipe): + """Sets up the needed objects for Tronstore, including message + factories, a synchronized pipe and store object, a thread pool for + handling requests, and some internal invariants. + """ + self.pipe = SyncPipe(pipe) + self.request_factory = StoreRequestFactory() + self.response_factory = StoreResponseFactory() + self.store_class = store.SyncStore(config) + self.thread_pool = TronstorePool(self.response_factory, self.pipe, + self.store_class) + self.is_shutdown = False + self.shutdown_req_id = None + self.config = config + + def _get_all_from_pipe(self): + """Gets all of the requests from the pipe, returning an array of serialized + requests (they still need to be decoded). + """ + requests = [] + while self.pipe.poll(): + requests.append(self.pipe.recv_bytes()) + return requests + + def _reconfigure(self, request): + """Reconfigures Tronstore by attempting to make a new store object + from the recieved configuration. If anything goes wrong, we revert + back to the old configuration. + """ + self.thread_pool.stop() + self.store_class.cleanup() try: - if pipe.poll(POLL_TIMEOUT): - requests = get_all_from_pipe(pipe) - requests = map(request_factory.from_msg, requests) - for request in requests: - if request.req_type == msg_enums.REQUEST_SHUTDOWN: - is_shutdown = True - shutdown_req_id = request.id - - elif request.req_type == msg_enums.REQUEST_CONFIG: - while len(running_threads) != 0 or not thread_queue.empty(): - time.sleep(0.5) - store_class.cleanup() - try: - # Try to set up with the new configuration - store_class = parse_config(request.data) - pipe.send_bytes(response_factory.build(True, request.id, '').serialized) - # request_factory.update_method(transport_method) - # response_factory.update_method(transport_method) - config = request.data - except: - # Failed, go back to what we had - store_class = parse_config(config) - # request_factory.update_method(transport_method) - # response_factory.update_method(transport_method) - pipe.send_bytes(response_factory.build(False, request.id, '').serialized) - - else: - request_thread = Thread(target=handle_request, - args=( - request, - store_class, - pipe, - response_factory, - save_lock, - restore_lock)) - request_thread.daemon = False - thread_queue.put(request_thread) - elif is_shutdown: - # We have to wait for all threads to clean up first. - while len(running_threads) != 0 or thread_pool.is_alive(): - time.sleep(0.5) - store_class.cleanup() - if shutdown_req_id: - pipe.send_bytes(response_factory.build(True, shutdown_req_id, '').serialized) - # TODO: Do we need a forceful kill? Can we just return here? - os._exit(0) - else: - # Did tron die? - try: - os.kill(os.getppid(), 0) - except: - is_shutdown = True - except IOError, e: - # Error #4 is a system interrupt, caused by ^C - if e.errno != 4: - raise + self.store_class = store.SyncStore(request.data) + self.thread_pool = TronstorePool(self.response_factory, self.pipe, + self.store_class) + self.thread_pool.start() + self.config = request.data + self.pipe.send_bytes(self.response_factory.build(True, request.id, '').serialized) + except: + self.store_class = store.SyncStore(self.config) + self.thread_pool = TronstorePool(self.response_factory, self.pipe, + self.store_class) + self.thread_pool.start() + self.pipe.send_bytes(self.response_factory.build(False, request.id, '').serialized) + + def _handle_request(self, request): + """Handle a request by either doing something with it ourselves + (in the case of shutdown/config), or passing it to a worker in the + thread pool (for save/restore). + """ + if request.req_type == msg_enums.REQUEST_SHUTDOWN: + self.is_shutdown = True + self.shutdown_req_id = request.id + + elif request.req_type == msg_enums.REQUEST_CONFIG: + self._reconfigure(request) + + else: + self.thread_pool.enqueue_work(request) + + def _shutdown(self): + """Shutdown Tronstore. Calls os._exit, and should only be called + once all work has been completed. + """ + self.thread_pool.stop() + self.store_class.cleanup() + if self.shutdown_req_id: + shutdown_resp = self.response_factory.build(True, self.shutdown_req_id, '') + self.pipe.send_bytes(shutdown_resp.serialized) + os._exit(0) # Hard exit- should kill everything. + + def main_loop(self): + """The main Tronstore event loop. Starts the thread pool and then + simply polls for requests until a shutdown request is recieved, after + which it cleans up and exits. + """ + self.thread_pool.start() + + while True: + try: + if self.pipe.poll(self.POLL_TIMEOUT): + requests = self._get_all_from_pipe() + requests = map(self.request_factory.from_msg, requests) + for request in requests: + self._handle_request(request) + + elif self.is_shutdown: + self._shutdown() + + else: + # Did tron die? + try: + os.kill(os.getppid(), 0) + except: + self.is_shutdown = True + except IOError, e: + # Error #4 is a system interrupt, caused by ^C + if e.errno != 4: + raise + + +def main(config, pipe): + """The main method to start Tronstore with. Simply takes the configuration + and pipe objects, and then registers some null signal handlers before + passing everything off to TronstoreMain. + """ + + _register_null_handlers() + tronstore = TronstoreMain(config, pipe) + tronstore.main_loop() From 8fefdb19c261096ce0a2d2706a2593b42ce4f8bb Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 30 Jul 2013 17:05:21 -0700 Subject: [PATCH 22/48] redid all broken unit tests --- tests/core/job_test.py | 6 +- tests/mcp_test.py | 18 +- tests/serialize/runstate/statemanager_test.py | 150 ++--- .../runstate/tronstore/parallelstore_test.py | 19 +- .../runstate/tronstore/process_test.py | 24 +- .../runstate/tronstore/store_test.py | 59 +- .../runstate/tronstore/tronstore_test.py | 558 ++++++++++-------- tron/serialize/runstate/statemanager.py | 8 +- tron/serialize/runstate/tronstore/store.py | 3 +- 9 files changed, 480 insertions(+), 365 deletions(-) diff --git a/tests/core/job_test.py b/tests/core/job_test.py index 400fc6a66..be78df2d2 100644 --- a/tests/core/job_test.py +++ b/tests/core/job_test.py @@ -104,7 +104,10 @@ def test_restore_state(self): job_runs = [mock.Mock(), mock.Mock()] state_data = ({'enabled': False, 'run_ids': [1, 2]}, run_data) - with mock.patch.object(self.job.job_runs, 'restore_state', return_value=job_runs): + with contextlib.nested( + mock.patch.object(self.job.job_runs, 'restore_state', return_value=job_runs), + mock.patch.object(self.job.job_runs, 'get_run_numbers', return_value=state_data[0]['run_ids']) + ): self.job.restore_state(state_data) assert not self.job.enabled @@ -118,6 +121,7 @@ def test_restore_state(self): self.job.context, self.job.node_pool ) + self.job.job_runs.get_run_numbers.assert_called_once_with() self.job.job_scheduler.restore_state.assert_called_once_with() self.job.event.ok.assert_called_with('restored') diff --git a/tests/mcp_test.py b/tests/mcp_test.py index 4e34827d5..6c967db74 100644 --- a/tests/mcp_test.py +++ b/tests/mcp_test.py @@ -20,10 +20,9 @@ class MasterControlProgramTestCase(TestCase): def setup_mcp(self): self.working_dir = tempfile.mkdtemp() self.config_path = tempfile.mkdtemp() - self.mcp = mcp.MasterControlProgram( + with mock.patch('tron.serialize.runstate.statemanager.StateChangeWatcher', autospec=True): + self.mcp = mcp.MasterControlProgram( self.working_dir, self.config_path) - self.mcp.state_watcher = mock.create_autospec( - statemanager.StateChangeWatcher) @teardown def teardown_mcp(self): @@ -64,7 +63,7 @@ def test_apply_config(self, mock_repo): assert_equal(len(self.mcp.apply_collection_config.mock_calls), 2) self.mcp.apply_notification_options.assert_called_with( master_config.notification_options) - mock_repo.update_from_config.assert_called_with(master_config.nodes, + mock_repo.update_from_config.assert_called_with(master_config.nodes, master_config.node_pools, master_config.ssh_options) self.mcp.build_job_scheduler_factory(master_config) @@ -98,11 +97,12 @@ class MasterControlProgramRestoreStateTestCase(TestCase): def setup_mcp(self): self.working_dir = tempfile.mkdtemp() self.config_path = tempfile.mkdtemp() - self.mcp = mcp.MasterControlProgram( - self.working_dir, self.config_path) - self.mcp.jobs = mock.create_autospec(job.JobCollection) - self.mcp.services = mock.create_autospec(service.ServiceCollection) - self.mcp.state_watcher = mock.create_autospec(statemanager.StateChangeWatcher) + with mock.patch('tron.serialize.runstate.statemanager.StateChangeWatcher', autospec=True): + self.mcp = mcp.MasterControlProgram( + self.working_dir, self.config_path) + self.mcp.jobs = mock.create_autospec(job.JobCollection) + self.mcp.services = mock.create_autospec(service.ServiceCollection) + self.mcp.state_watcher = mock.create_autospec(statemanager.StateChangeWatcher) @teardown def teardown_mcp(self): diff --git a/tests/serialize/runstate/statemanager_test.py b/tests/serialize/runstate/statemanager_test.py index 3586fcd07..891193b26 100644 --- a/tests/serialize/runstate/statemanager_test.py +++ b/tests/serialize/runstate/statemanager_test.py @@ -1,72 +1,15 @@ -import os import mock import contextlib -from testify import TestCase, assert_equal, setup, run, setup_teardown +from testify import TestCase, assert_equal, setup, run from tests.assertions import assert_raises from tests.testingutils import autospec_method -from tron.config import schema from tron.serialize import runstate -from tron.serialize.runstate.shelvestore import ShelveStateStore from tron.serialize.runstate.statemanager import PersistentStateManager, StateChangeWatcher from tron.serialize.runstate.statemanager import StateSaveBuffer from tron.serialize.runstate.statemanager import StateMetadata from tron.serialize.runstate.statemanager import PersistenceStoreError from tron.serialize.runstate.statemanager import VersionMismatchError -from tron.serialize.runstate.statemanager import PersistenceManagerFactory -from tron.serialize.runstate.statemanager import NullStateManager - - -class PersistenceManagerFactoryTestCase(TestCase): - - @setup_teardown - def setup_factory_and_enumerate(self): - self.mock_buffer_size = 25 - self.mock_config = mock.Mock(buffer_size=self.mock_buffer_size) - with contextlib.nested( - mock.patch('tron.serialize.runstate.statemanager.ParallelStore', - autospec=True), - mock.patch('%s.%s' % (PersistentStateManager.__module__, PersistentStateManager.__name__), - autospec=True), - mock.patch('%s.%s' % (StateSaveBuffer.__module__, StateSaveBuffer.__name__), - autospec=True) - ) as (self.parallel_patch, self.state_patch, self.buffer_patch): - yield - - def test_from_config_all_valid_enum_types(self): - for store_type in schema.StatePersistenceTypes.values: - self.mock_config.configure_mock(store_type=store_type) - for transport_method in schema.StateTransportTypes.values: - self.mock_config.configure_mock(transport_method=transport_method) - if store_type in ('sql', 'mongo'): - self.mock_config.configure_mock(db_store_method=transport_method) - - assert_equal(PersistenceManagerFactory.from_config(self.mock_config), - self.state_patch(self.parallel_patch, self.buffer_patch)) - self.parallel_patch.assert_called_with(self.mock_config) - self.buffer_patch.assert_called_with(self.mock_buffer_size) - - def test_from_config_invalid_store_type(self): - self.mock_config.configure_mock(store_type='play_the_game') - for transport_method in schema.StateTransportTypes.values: - self.mock_config.configure_mock(transport_method=transport_method) - assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) - - def test_from_config_invalid_transport_type(self): - self.mock_config.configure_mock(transport_method='ghosts_cant_eat') - for store_type in schema.StatePersistenceTypes.values: - self.mock_config.configure_mock(store_type=store_type) - if store_type in ('sql', 'mongo'): - self.mock_config.configure_mock(db_store_method='json') - assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) - - def test_from_config_invalid_db_store_method(self): - self.mock_config.configure_mock(db_store_method='im_running_out_of_strs') - for store_type in ('sql', 'mongo'): - self.mock_config.configure_mock(store_type=store_type) - for transport_method in schema.StateTransportTypes.values: - self.mock_config.configure_mock(transport_method=transport_method) - assert_raises(PersistenceStoreError, PersistenceManagerFactory.from_config, self.mock_config) class StateMetadataTestCase(TestCase): @@ -113,13 +56,19 @@ class PersistentStateManagerTestCase(TestCase): @setup def setup_manager(self): - self.store = mock.Mock() - self.store.build_key.side_effect = lambda t, i: '%s%s' % (t, i) - self.buffer = StateSaveBuffer(1) - self.manager = PersistentStateManager(self.store, self.buffer) + with mock.patch('tron.serialize.runstate.statemanager.ParallelStore', autospec=True) \ + as self.store_patch: + self.store = self.store_patch.return_value + self.build_patch = mock.Mock(side_effect=lambda t, i: '%s%s' % (t, i)) + self.store_patch.return_value.configure_mock(build_key=self.build_patch) + self.buffer = StateSaveBuffer(1) + self.manager = PersistentStateManager() + self.manager._buffer = self.buffer def test__init__(self): - assert_equal(self.manager._impl, self.store) + self.store_patch.assert_called_once_with() + self.build_patch.assert_called_once_with(runstate.MCP_STATE, StateMetadata.name) + assert_equal(self.manager.metadata_key, self.manager._impl.build_key(runstate.MCP_STATE, StateMetadata.name)) def test_keys_for_items(self): names = ['namea', 'nameb'] @@ -177,23 +126,43 @@ def test_disabled_nested(self): pass assert not self.manager.enabled - def test_update_config(self): - new_config = mock.Mock() - fake_return = 'reelin_in_the_years' - self.store.load_config.configure_mock(return_value=fake_return) - with mock.patch.object(self.manager, '_save_from_buffer') as save_patch: - assert_equal(self.manager.update_from_config(new_config), fake_return) + def test_update_config_success(self): + new_config = mock.Mock(buffer_size=5) + self.store.load_config.configure_mock(return_value=True) + with contextlib.nested( + mock.patch.object(self.manager, '_save_from_buffer'), + mock.patch('tron.serialize.runstate.statemanager.StateSaveBuffer', autospec=True) + ) as (save_patch, buffer_patch): + assert_equal(self.manager.update_from_config(new_config), True) save_patch.assert_called_once_with() self.store.load_config.assert_called_once_with(new_config) + buffer_patch.assert_called_once_with(new_config.buffer_size) + + def test_update_config_failure(self): + new_config = mock.Mock(buffer_size=5) + self.store.load_config.configure_mock(return_value=False) + with contextlib.nested( + mock.patch.object(self.manager, '_save_from_buffer'), + mock.patch('tron.serialize.runstate.statemanager.StateSaveBuffer', autospec=True) + ) as (save_patch, buffer_patch): + assert_equal(self.manager.update_from_config(new_config), False) + save_patch.assert_called_once_with() + self.store.load_config.assert_called_once_with(new_config) + assert not buffer_patch.called class StateChangeWatcherTestCase(TestCase): @setup def setup_watcher(self): - self.watcher = StateChangeWatcher() - self.state_manager = mock.create_autospec(PersistentStateManager) - self.watcher.state_manager = self.state_manager + with mock.patch('tron.serialize.runstate.statemanager.PersistentStateManager', autospec=True) \ + as self.persistence_patch: + self.watcher = StateChangeWatcher() + self.state_manager = mock.create_autospec(PersistentStateManager) + self.watcher.state_manager = self.state_manager + + def test__init__(self): + self.persistence_patch.assert_called_once_with() def test_update_from_config_no_change(self): self.watcher.config = state_config = mock.Mock() @@ -202,31 +171,40 @@ def test_update_from_config_no_change(self): assert_equal(self.watcher.state_manager, self.state_manager) assert not self.watcher.shutdown.mock_calls - @mock.patch('tron.serialize.runstate.statemanager.PersistenceManagerFactory', - autospec=True) - def test_update_from_config_no_state_manager(self, mock_factory): - state_config = mock.Mock() - self.watcher.state_manager = NullStateManager + def test_update_from_config_success(self): + state_config = mock.Mock(store_type="shelve") assert self.watcher.update_from_config(state_config) assert_equal(self.watcher.config, state_config) - assert_equal(self.watcher.state_manager, - mock_factory.from_config.return_value) - mock_factory.from_config.assert_called_with(state_config) + self.state_manager.update_from_config.assert_called_once_with(state_config) - def test_update_from_config_with_state_manager_success(self): - state_config = mock.Mock() - assert self.watcher.update_from_config(state_config) + def test_update_from_config_failure_same_config(self): + state_config = self.watcher.config + assert not self.watcher.update_from_config(state_config) assert_equal(self.watcher.config, state_config) - self.state_manager.update_from_config.assert_called_once_with(state_config) + assert not self.state_manager.update_from_config.called - def test_update_from_config_failure(self): + def test_update_from_config_failure_from_state_manager(self): self.state_manager.update_from_config.configure_mock(return_value=False) state_config = self.watcher.config - fake_config = mock.Mock() + fake_config = mock.Mock(store_type="shelve") assert not self.watcher.update_from_config(fake_config) assert_equal(self.watcher.config, state_config) self.state_manager.update_from_config.assert_called_once_with(fake_config) + def test_update_from_config_failure_bad_store_type(self): + state_config = self.watcher.config + fake_config = mock.Mock(store_type="hue_hue_hue") + assert_raises(PersistenceStoreError, self.watcher.update_from_config, fake_config) + assert_equal(self.watcher.config, state_config) + assert not self.state_manager.update_from_config.called + + def test_update_from_config_failure_bad_db_type(self): + state_config = self.watcher.config + fake_config = mock.Mock(store_type="sql", store_method="make_it_rain") + assert_raises(PersistenceStoreError, self.watcher.update_from_config, fake_config) + assert_equal(self.watcher.config, state_config) + assert not self.state_manager.update_from_config.called + def test_save_job(self): mock_job = mock.Mock() self.watcher.save_job(mock_job) diff --git a/tests/serialize/runstate/tronstore/parallelstore_test.py b/tests/serialize/runstate/tronstore/parallelstore_test.py index 9139c119c..05cf6091f 100644 --- a/tests/serialize/runstate/tronstore/parallelstore_test.py +++ b/tests/serialize/runstate/tronstore/parallelstore_test.py @@ -20,11 +20,12 @@ def setup_store(self): ) with mock.patch('tron.serialize.runstate.tronstore.parallelstore.StoreProcessProtocol', autospec=True) \ as (self.process_patch): - self.store = ParallelStore(self.config) + self.store = ParallelStore() yield def test__init__(self): - self.process_patch.assert_called_once_with(self.config, self.store.response_factory) + self.process_patch.assert_called_once_with() + assert self.store.request_factory def test_build_key(self): key_type = runstate.JOB_STATE @@ -85,34 +86,26 @@ def test_load_config_success(self): new_config = mock.Mock() config_req = mock.Mock() with contextlib.nested( - mock.patch.object(self.store.request_factory, 'update_method'), - mock.patch.object(self.store.response_factory, 'update_method'), mock.patch.object(self.store.process, 'update_config'), mock.patch.object(self.store.request_factory, 'build', return_value=config_req), mock.patch.object(self.store.process, 'send_request_get_response', return_value=mock.Mock(success=True)) - ) as (request_patch, response_patch, update_patch, build_patch, send_patch): + ) as (update_patch, build_patch, send_patch): self.store.load_config(new_config) build_patch.assert_called_once_with(msg_enums.REQUEST_CONFIG, '', new_config) send_patch.assert_called_once_with(config_req) - request_patch.assert_called_once_with(new_config.transport_method) - response_patch.assert_called_once_with(new_config.transport_method) - update_patch.assert_called_once_with(new_config, config_req) + update_patch.assert_called_once_with(new_config) def test_load_config_failure(self): new_config = mock.Mock() config_req = mock.Mock() with contextlib.nested( - mock.patch.object(self.store.request_factory, 'update_method'), - mock.patch.object(self.store.response_factory, 'update_method'), mock.patch.object(self.store.process, 'update_config'), mock.patch.object(self.store.request_factory, 'build', return_value=config_req), mock.patch.object(self.store.process, 'send_request_get_response', return_value=mock.Mock(success=False)) - ) as (request_patch, response_patch, update_patch, build_patch, send_patch): + ) as (update_patch, build_patch, send_patch): self.store.load_config(new_config) build_patch.assert_called_once_with(msg_enums.REQUEST_CONFIG, '', new_config) send_patch.assert_called_once_with(config_req) assert not update_patch.called - assert not request_patch.called - assert not response_patch.called diff --git a/tests/serialize/runstate/tronstore/process_test.py b/tests/serialize/runstate/tronstore/process_test.py index d9cb8a4e9..503d6913e 100644 --- a/tests/serialize/runstate/tronstore/process_test.py +++ b/tests/serialize/runstate/tronstore/process_test.py @@ -17,29 +17,23 @@ def setup_process(self): mock.patch('tron.serialize.runstate.tronstore.process.Process', autospec=True), mock.patch('tron.serialize.runstate.tronstore.process.Pipe', - new=pipe_return) - ) as (self.process_patch, self.pipe_setup_patch): - self.config = mock.Mock( - name='test_config', - transport_method='pickle', - store_type='shelve', - connection_details=None, - db_store_method=None, - buffer_size=1 - ) - self.factory = mock.Mock() - self.process = StoreProcessProtocol(self.config, self.factory) + new=pipe_return), + mock.patch('tron.serialize.runstate.tronstore.process.StoreResponseFactory') + ) as (self.process_patch, self.pipe_setup_patch, self.factory_patch): + self.factory = self.factory_patch.return_value + self.process = StoreProcessProtocol() yield def test__init__(self): - assert_equal(self.process.response_factory, self.factory) - assert_equal(self.process.config, self.config) + assert not self.process.config + self.factory_patch.assert_called_once_with() assert_equal(self.process.orphaned_responses, {}) assert not self.process.is_shutdown + assert self.process.pipe def test_start_process(self): self.pipe_setup_patch.assert_called_once_with() - self.process_patch.assert_called_once_with(target=tronstore.main, args=(self.config, self.test_pipe_b)) + self.process_patch.assert_called_once_with(target=tronstore.main, args=(self.process.config, self.test_pipe_b)) assert self.process_patch.daemon self.process.process.start.assert_called_once_with() diff --git a/tests/serialize/runstate/tronstore/store_test.py b/tests/serialize/runstate/tronstore/store_test.py index c838c1a22..159fd3c05 100644 --- a/tests/serialize/runstate/tronstore/store_test.py +++ b/tests/serialize/runstate/tronstore/store_test.py @@ -1,8 +1,10 @@ import os import shelve import tempfile +import mock +import contextlib from testify import TestCase, run, setup, assert_equal, teardown -from tron.serialize.runstate.tronstore.store import ShelveStore, SQLStore, MongoStore, YamlStore +from tron.serialize.runstate.tronstore.store import ShelveStore, SQLStore, MongoStore, YamlStore, SyncStore, NullStore from tron.serialize.runstate.tronstore.transport import JSONTransport from tron.serialize import runstate @@ -240,5 +242,60 @@ def test_save(self): actual = yaml.load(fh) assert_equal(actual, expected) + +class SyncStoreTestCase(TestCase): + + @setup + def setup_sync_store(self): + self.fake_config = mock.Mock( + name='we_must_be_swift_as_a_coursing_river', + store_type='with_all_the_force_of_a_great_typhoon', + connection_details='with_all_the_strength_of_a_raging_fire', + db_store_method='mysterious_as_the_dark_side_of_the_moon') + self.store_class = mock.Mock() + with contextlib.nested( + mock.patch.object(runstate.tronstore.store, 'build_store', return_value=self.store_class), + mock.patch('tron.serialize.runstate.tronstore.store.Lock', autospec=True) + ) as (self.build_patch, self.lock_patch): + self.store = SyncStore(self.fake_config) + self.lock = self.lock_patch.return_value + + def test__init__(self): + self.lock_patch.assert_called_once_with() + self.build_patch.assert_called_once_with( + self.fake_config.name, + self.fake_config.store_type, + self.fake_config.connection_details, + self.fake_config.db_store_method + ) + assert_equal(self.store_class, self.store.store) + + def test__init__null_config(self): + store = SyncStore(None) + assert isinstance(store.store, NullStore) + + def test_save(self): + fake_arg = 'catch_a_ride' + fake_kwarg = 'no_refunds' + self.store.save(fake_arg, fake_kwarg=fake_kwarg) + self.lock.__enter__.assert_called_once_with() + self.lock.__exit__.assert_called_once_with(None, None, None) + self.store_class.save.assert_called_once_with(fake_arg, fake_kwarg=fake_kwarg) + + def test_restore(self): + fake_arg = 'catch_a_ride' + fake_kwarg = 'no_refunds' + self.store.restore(fake_arg, fake_kwarg=fake_kwarg) + self.lock.__enter__.assert_called_once_with() + self.lock.__exit__.assert_called_once_with(None, None, None) + self.store_class.restore.assert_called_once_with(fake_arg, fake_kwarg=fake_kwarg) + + def test_cleanup(self): + self.store.cleanup() + self.lock.__enter__.assert_called_once_with() + self.lock.__exit__.assert_called_once_with(None, None, None) + self.store_class.cleanup.assert_called_once_with() + + if __name__ == "__main__": run() diff --git a/tests/serialize/runstate/tronstore/tronstore_test.py b/tests/serialize/runstate/tronstore/tronstore_test.py index 6363d6049..494687cd8 100644 --- a/tests/serialize/runstate/tronstore/tronstore_test.py +++ b/tests/serialize/runstate/tronstore/tronstore_test.py @@ -1,8 +1,8 @@ import contextlib import mock import signal -from Queue import Queue -from testify import TestCase, assert_equal, assert_raises, setup_teardown, setup +from Queue import Queue, Empty +from testify import TestCase, assert_equal, assert_raises, setup_teardown, setup, run from tron.serialize.runstate.tronstore import tronstore, msg_enums @@ -13,16 +13,9 @@ def setup_main(self): self.config = mock.Mock() self.pipe = mock.Mock() self.store_class = mock.Mock() - self.trans_method = mock.Mock() - self.mock_thread = mock.Mock(is_alive=lambda: False) + self.thread_pool = mock.Mock() self.request_factory = mock.Mock() self.response_factory = mock.Mock() - self.lock = mock.Mock() - self.queue = mock.Mock() - - def poll_patch(timeout): - return False if tronstore.is_shutdown else True - self.pipe.poll = poll_patch def echo_single_request(request): return request @@ -35,171 +28,200 @@ def raise_to_exit(exitcode): raise SystemError with contextlib.nested( - mock.patch.object(tronstore, 'parse_config', - return_value=(self.store_class, self.trans_method)), - mock.patch('tron.serialize.runstate.tronstore.tronstore.Thread', - new=mock.Mock(return_value=self.mock_thread)), - mock.patch.object(signal, 'signal'), - mock.patch.object(tronstore, 'get_all_from_pipe', - side_effect=echo_requests), + mock.patch('tron.serialize.runstate.tronstore.store.SyncStore', + new=mock.Mock(return_value=self.store_class)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.TronstorePool', + new=mock.Mock(return_value=self.thread_pool)), + mock.patch('tron.serialize.runstate.tronstore.tronstore.SyncPipe', + new=mock.Mock(return_value=self.pipe)), mock.patch('tron.serialize.runstate.tronstore.tronstore.StoreRequestFactory', new=mock.Mock(return_value=self.request_factory)), mock.patch('tron.serialize.runstate.tronstore.tronstore.StoreResponseFactory', new=mock.Mock(return_value=self.response_factory)), - mock.patch('tron.serialize.runstate.tronstore.tronstore.Lock', - new=mock.Mock(return_value=self.lock)), - mock.patch('tron.serialize.runstate.tronstore.tronstore.Queue', - new=mock.Mock(return_value=self.queue)), mock.patch.object(tronstore.os, '_exit', - side_effect=raise_to_exit) + autospec=True) ) as ( - self.parse_patch, + self.store_patch, self.thread_patch, - self.signal_patch, - self.get_all_patch, + self.pipe_patch, self.request_patch, self.response_patch, - self.lock_patch, - self.queue_patch, self.exit_patch ): + self.main = tronstore.TronstoreMain(self.config, self.pipe) yield - def assert_main_startup(self): - self.parse_patch.assert_any_call(self.config) - self.request_patch.assert_called_once_with(self.trans_method) - self.response_patch.assert_called_once_with(self.trans_method) - assert_equal(self.lock_patch.call_count, 2) - self.queue_patch.assert_called_once_with() - self.thread_patch.assert_any_call(target=tronstore.thread_starter, args=(self.queue, [])) - self.mock_thread.start.assert_called_once_with() - self.signal_patch.assert_any_call(signal.SIGINT, tronstore._discard_signal) - self.signal_patch.assert_any_call(signal.SIGHUP, tronstore._discard_signal) - self.signal_patch.assert_any_call(signal.SIGTERM, tronstore._discard_signal) - self.exit_patch.assert_called_once_with(0) + def test__init__(self): + self.store_patch.assert_called_once_with(self.config) + self.request_patch.assert_called_once_with() + self.pipe_patch.assert_called_once_with(self.pipe) + self.response_patch.assert_called_once_with() + self.thread_patch.assert_called_once_with(self.response_factory, self.pipe, self.store_class) + assert_equal(self.main.config, self.config) + assert not self.main.is_shutdown + assert not self.main.shutdown_req_id + + def test_get_all_from_pipe(self): + fake_data = 'fuego' + self.pipe.recv_bytes = mock.Mock(return_value=fake_data) + self.pipe.poll = mock.Mock(side_effect=iter([True, False])) + assert_equal(self.main._get_all_from_pipe(), [fake_data]) + self.pipe.recv_bytes.assert_called_once_with() + assert_equal(self.pipe.poll.call_count, 2) - def test_shutdown_request(self): + def test_reconfigure_success(self): fake_id = 77 - self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_id)] - assert_raises(SystemError, tronstore.main, self.config, self.pipe) + fake_data = mock.Mock() + request = mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id, data=fake_data) - self.assert_main_startup() - assert tronstore.is_shutdown - self.get_all_patch.assert_called_once_with(self.pipe) + self.main._reconfigure(request) + self.thread_pool.stop.assert_called_once_with() + self.thread_pool.start.assert_called_once_with() self.store_class.cleanup.assert_called_once_with() + self.store_patch.assert_any_call(fake_data) + self.thread_patch.assert_any_call(self.response_factory, self.pipe, self.store_class) + assert_equal(self.thread_patch.call_count, 2) + assert_equal(self.main.config, fake_data) self.response_factory.build.assert_called_once_with(True, fake_id, '') self.pipe.send_bytes.assert_called_once_with(self.response_factory.build().serialized) - def test_config_request_success(self): - fake_id = 77 - fake_shutdown_id = 88 - self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), - mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - assert_raises(SystemError, tronstore.main, self.config, self.pipe) - - self.assert_main_startup() - assert tronstore.is_shutdown - assert_equal(self.store_class.cleanup.call_count, 2) - self.store_class.cleanup.assert_any_call() - assert_equal(self.parse_patch.call_count, 2) - self.queue.empty.assert_called_once_with() - self.response_factory.build.assert_any_call(True, fake_id, '') - self.pipe.send_bytes.assert_any_call(self.response_factory.build().serialized) - self.request_factory.update_method.assert_called_once_with(self.trans_method) - self.response_factory.update_method.assert_called_once_with(self.trans_method) - - def test_config_request_exception(self): + def test_reconfigure_failure(self): fake_id = 77 - fake_shutdown_id = 88 - some_fake_store = mock.Mock() - some_fake_trans = mock.Mock() - self.requests = [mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id), - mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - item_iter = iter([(self.store_class, self.trans_method), 'breakit', - (some_fake_store, some_fake_trans)]) - self.parse_patch.configure_mock(side_effect=item_iter) - assert_raises(SystemError, tronstore.main, self.config, self.pipe) - - self.assert_main_startup() - assert tronstore.is_shutdown + fake_data = mock.Mock() + request = mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id, data=fake_data) + self.store_patch.configure_mock(side_effect=iter([SystemError, lambda x: None])) + + self.main._reconfigure(request) + assert_equal(self.store_patch.call_count, 3) + self.store_patch.assert_any_call(fake_data) + self.store_patch.assert_any_call(self.config) + self.thread_patch.assert_any_call(self.response_factory, self.pipe, + self.store_class) + assert_equal(self.thread_patch.call_count, 2) + self.thread_pool.stop.assert_called_once_with() self.store_class.cleanup.assert_called_once_with() - some_fake_store.cleanup.assert_called_once_with() - assert_equal(self.parse_patch.call_count, 3) - self.queue.empty.assert_called_once_with() - self.response_factory.build.assert_any_call(False, fake_id, '') - self.pipe.send_bytes.assert_any_call(self.response_factory.build().serialized) - self.request_factory.update_method.assert_called_once_with(some_fake_trans) - self.response_factory.update_method.assert_called_once_with(some_fake_trans) - - def test_save_request(self): + self.thread_pool.start.assert_called_once_with() + self.response_factory.build.assert_called_once_with(False, fake_id, '') + self.pipe.send_bytes.assert_called_once_with(self.response_factory.build().serialized) + + def test_handle_request_save(self): fake_id = 77 - fake_shutdown_id = 88 - self.requests = [mock.Mock(req_type=msg_enums.REQUEST_SAVE, id=fake_id), - mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - assert_raises(SystemError, tronstore.main, self.config, self.pipe) - - self.assert_main_startup() - assert tronstore.is_shutdown - self.thread_patch.assert_any_call(target=tronstore.handle_request, - args=( - self.requests[0], - self.store_class, - self.pipe, - self.response_factory, - self.lock, - self.lock)) - self.queue.put.assert_called_once_with(self.mock_thread) + request = mock.Mock(req_type=msg_enums.REQUEST_SAVE, id=fake_id) + self.main._handle_request(request) + self.thread_pool.enqueue_work.assert_called_once_with(request) - def test_restore_request(self): + def test_handle_request_restore(self): fake_id = 77 - fake_shutdown_id = 88 - self.requests = [mock.Mock(req_type=msg_enums.REQUEST_RESTORE, id=fake_id), - mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_shutdown_id)] - assert_raises(SystemError, tronstore.main, self.config, self.pipe) - - self.assert_main_startup() - assert tronstore.is_shutdown - self.thread_patch.assert_any_call(target=tronstore.handle_request, - args=( - self.requests[0], - self.store_class, - self.pipe, - self.response_factory, - self.lock, - self.lock)) - self.queue.put.assert_called_once_with(self.mock_thread) + request = mock.Mock(req_type=msg_enums.REQUEST_RESTORE, id=fake_id) + self.main._handle_request(request) + self.thread_pool.enqueue_work.assert_called_once_with(request) + + def test_handle_request_shutdown(self): + fake_id = 77 + request = mock.Mock(req_type=msg_enums.REQUEST_SHUTDOWN, id=fake_id) + self.main._handle_request(request) + assert self.main.is_shutdown + assert_equal(fake_id, self.main.shutdown_req_id) + + def test_handle_request_config(self): + fake_id = 77 + request = mock.Mock(req_type=msg_enums.REQUEST_CONFIG, id=fake_id) + with mock.patch.object(self.main, '_reconfigure') as reconf_patch: + self.main._handle_request(request) + reconf_patch.assert_called_once_with(request) + + def test_shutdown_has_id(self): + fake_id = 77 + self.main.shutdown_req_id = fake_id + self.main._shutdown() + self.thread_pool.stop.assert_called_once_with() + self.store_class.cleanup.assert_called_once_with() + self.response_factory.build.assert_called_once_with(True, fake_id, '') + self.pipe.send_bytes.assert_called_once_with(self.response_factory.build().serialized) + self.exit_patch.assert_called_once_with(0) + + def test_shutdown_no_id(self): + self.main.shutdown_req_id = None + self.main._shutdown() + self.thread_pool.stop.assert_called_once_with() + self.store_class.cleanup.assert_called_once_with() + assert not self.response_factory.build.called + assert not self.pipe.send_bytes.called + self.exit_patch.assert_called_once_with(0) + + def test_main_loop_handle_requests(self): + self.main.is_shutdown = False + self.pipe.poll = mock.Mock(return_value=True) + requests = [mock.Mock(), mock.Mock()] + self.request_factory.from_msg = mock.Mock(side_effect=lambda x: x) + with contextlib.nested( + mock.patch.object(self.main, '_get_all_from_pipe', return_value=requests), + mock.patch.object(self.main, '_handle_request', + side_effect=iter([None, SystemError])) + ) as (all_patch, handle_patch): + assert_raises(SystemError, self.main.main_loop) + self.thread_pool.start.assert_called_once_with() + self.pipe.poll.assert_called_once_with(self.main.POLL_TIMEOUT) + all_patch.assert_called_once_with() + self.request_factory.from_msg.assert_has_calls( + [mock.call(requests[i]) for i in xrange(len(requests))]) + handle_patch.assert_has_calls( + [mock.call(requests[i]) for i in xrange(len(requests))]) + + def test_main_loop_is_shutdown(self): + self.main.is_shutdown = True + self.pipe.poll.configure_mock(return_value=False) + with mock.patch.object(self.main, '_shutdown', + side_effect=SystemError) as shutdown_patch: + assert_raises(SystemError, self.main.main_loop) + self.thread_pool.start.assert_called_once_with() + self.pipe.poll.assert_called_once_with(self.main.POLL_TIMEOUT) + shutdown_patch.assert_called_once_with() + + def test_main_loop_trond_check(self): + fake_id = 77 + self.main.is_shutdown = False + self.pipe.poll = mock.Mock(side_effect=iter([False, SystemError])) + with contextlib.nested( + mock.patch.object(tronstore.os, 'kill', side_effect=TypeError), + mock.patch.object(tronstore.os, 'getppid', return_value=fake_id) + ) as (kill_patch, ppid_patch): + assert_raises(SystemError, self.main.main_loop) + assert self.main.is_shutdown + ppid_patch.assert_called_once_with() + kill_patch.assert_called_once_with(fake_id, 0) + assert_equal(self.pipe.poll.call_count, 2) -class TronstoreHandleRequestTestCase(TestCase): +class TronstoreHandleRequestsTestCase(TestCase): @setup def setup_args(self): - self.store_class = mock.Mock() - self.pipe = mock.Mock() - self.factory = mock.Mock() - self.save_lock = mock.MagicMock() - self.restore_lock = mock.MagicMock() + self.queue = mock.Mock() + self.store_class = mock.Mock() + self.pipe = mock.Mock() + self.factory = mock.Mock() + self.do_work = mock.Mock(val=False) - def test_handle_request_save(self): + self.queue.empty.configure_mock(side_effect=iter([False, True])) + + def test_handle_requests_save(self): fake_id = 3090 request_data = ('fantastic', 'voyage') data_type = 'lakeside' - fake_success = 'eaten_by_a_gru' - self.store_class.save = mock.Mock(return_value=fake_success) request = mock.Mock(req_type=msg_enums.REQUEST_SAVE, data=request_data, data_type=data_type, id=fake_id) + self.queue.get.configure_mock(return_value=request) - tronstore.handle_request(request, self.store_class, self.pipe, - self.factory, self.save_lock, self.restore_lock) + tronstore.handle_requests(self.queue, self.factory, self.pipe, + self.store_class, self.do_work) - self.save_lock.__enter__.assert_called_once_with() - self.save_lock.__exit__.assert_called_once_with(None, None, None) self.store_class.save.assert_called_once_with(request_data[0], request_data[1], data_type) - self.factory.build.assert_called_once_with(fake_success, fake_id, '') - self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + assert_equal(self.queue.empty.call_count, 2) + self.queue.get.assert_called_once_with(block=True, timeout=1.0) - def test_handle_request_restore(self): + def test_handle_requests_restore(self): fake_id = 53045 request_data = 'edgeworth' fake_success = ('steel_samurai_fan', 'or_maybe_its_ironic') @@ -207,125 +229,193 @@ def test_handle_request_restore(self): self.store_class.restore = mock.Mock(return_value=fake_success) request = mock.Mock(req_type=msg_enums.REQUEST_RESTORE, data=request_data, data_type=data_type, id=fake_id) + self.queue.get.configure_mock(return_value=request) - tronstore.handle_request(request, self.store_class, self.pipe, - self.factory, self.save_lock, self.restore_lock) + tronstore.handle_requests(self.queue, self.factory, self.pipe, + self.store_class, self.do_work) - self.restore_lock.__enter__.assert_called_once_with() - self.restore_lock.__exit__.assert_called_once_with(None, None, None) self.store_class.restore.assert_called_once_with(request_data, data_type) self.factory.build.assert_called_once_with(fake_success[0], fake_id, fake_success[1]) self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + assert_equal(self.queue.empty.call_count, 2) + self.queue.get.assert_called_once_with(block=True, timeout=1.0) - def test_handle_request_other(self): + def test_handle_requests_other(self): fake_id = 1234567890 request = mock.Mock(req_type='not_actually_a_request', id=fake_id) + self.queue.get.configure_mock(return_value=request) - tronstore.handle_request(request, self.store_class, self.pipe, - self.factory, self.save_lock, self.restore_lock) + tronstore.handle_requests(self.queue, self.factory, self.pipe, + self.store_class, self.do_work) self.factory.build.assert_called_once_with(False, fake_id, '') self.pipe.send_bytes.assert_called_once_with(self.factory.build().serialized) + assert_equal(self.queue.empty.call_count, 2) + self.queue.get.assert_called_once_with(block=True, timeout=1.0) + def test_handle_requests_cont_on_empty(self): + self.queue.get.configure_mock(side_effect=Empty) -class TronstoreOtherTestCase(TestCase): + tronstore.handle_requests(self.queue, self.factory, self.pipe, + self.store_class, self.do_work) - def test_parse_config(self): - fake_config = mock.Mock( - name='yo_earl', - transport_method='what', - store_type='you\'re fired', - connection_details='HNNNNNNLLLLLGGG', - db_store_method='one_too_many_lines') - fake_store = 'lady_madonna' - with mock.patch.object(tronstore.store, 'build_store', - return_value=fake_store) as build_patch: - assert_equal(tronstore.parse_config(fake_config), (fake_store, fake_config.transport_method)) - build_patch.assert_called_once_with( - fake_config.name, - fake_config.store_type, - fake_config.connection_details, - fake_config.db_store_method) + assert_equal(self.queue.empty.call_count, 2) + self.queue.get.assert_called_once_with(block=True, timeout=1.0) + assert not self.pipe.send_bytes.called - def test_get_all_from_pipe(self): - fake_data = 'fuego' - pipe = mock.Mock() - pipe.recv_bytes = mock.Mock(return_value=fake_data) - pipe.poll = mock.Mock(side_effect=iter([True, False])) - assert_equal(tronstore.get_all_from_pipe(pipe), [fake_data]) - pipe.recv_bytes.assert_called_once_with() - assert_equal(pipe.poll.call_count, 2) +class TronstoreOtherTestCase(TestCase): -class TronstoreThreadStarterTestCase(TestCase): + def test_main(self): + config = mock.Mock() + pipe = mock.Mock() + with contextlib.nested( + mock.patch.object(tronstore, '_register_null_handlers'), + mock.patch('tron.serialize.runstate.tronstore.tronstore.TronstoreMain', autospec=True) + ) as (handler_patch, tronstore_patch): + tronstore.main(config, pipe) + handler_patch.assert_called_once_with() + tronstore_patch.assert_called_once_with(config, pipe) + tronstore_patch.return_value.main_loop.assert_called_once_with() + + def test_register_null_handlers(self): + with mock.patch.object(tronstore.signal, 'signal') as signal_patch: + tronstore._register_null_handlers() + signal_patch.assert_any_call(signal.SIGINT, tronstore._discard_signal) + signal_patch.assert_any_call(signal.SIGTERM, tronstore._discard_signal) + signal_patch.assert_any_call(signal.SIGHUP, tronstore._discard_signal) + + +class PoolBoolTestCase(TestCase): + + def test__init__(self): + poolbool = tronstore.PoolBool() + assert poolbool._val + assert poolbool.val + assert poolbool.value + + def test__init__invalid(self): + assert_raises(TypeError, tronstore.PoolBool, 'frue') + + def test__init__false(self): + poolbool = tronstore.PoolBool(False) + assert not poolbool._val + assert not poolbool.val + assert not poolbool.value + + def test_set(self): + poolbool = tronstore.PoolBool(False) + poolbool.set(True) + assert poolbool._val + assert poolbool.val + assert poolbool.value + + def test_set_invalid(self): + poolbool = tronstore.PoolBool(False) + assert_raises(TypeError, poolbool.set, 'tralse') + assert not poolbool._val + assert not poolbool.val + assert not poolbool.value + + +class SyncPipeTestCase(TestCase): + + @setup + def setup_pipe(self): + self.pipe = mock.Mock() + self.sync = tronstore.SyncPipe(self.pipe) + self.sync.lock = mock.MagicMock() + + def test__init__(self): + assert_equal(self.pipe, self.sync.pipe) + + def test_poll(self): + fake_arg = 'arrrrrrg' + fake_kwarg = 'no_dont_do_it_nishbot_we_love_you' + self.sync.poll(fake_arg, fake_kwarg=fake_kwarg) + self.pipe.poll.assert_called_once_with(fake_arg, fake_kwarg=fake_kwarg) + assert not self.sync.lock.__enter__.called + assert not self.sync.lock.lock.called + + def test_send_bytes(self): + fake_arg = 'makin_bacon' + fake_kwarg = 'hioh_its_mnc' + fake_return = 'churros' + self.pipe.send_bytes.configure_mock(return_value=fake_return) + assert_equal(self.sync.send_bytes(fake_arg, fake_kwarg=fake_kwarg), fake_return) + self.pipe.send_bytes.assert_called_once_with(fake_arg, fake_kwarg=fake_kwarg) + self.sync.lock.__enter__.assert_called_once_with() + self.sync.lock.__exit__.assert_called_once_with(None, None, None) + + def test_recv_bytes(self): + fake_arg = 'hey_can_i_have_root' + fake_kwarg = 'pls' + fake_return = 'PPFFFFFFFFAHAHAHAHAHHA' + self.pipe.recv_bytes.configure_mock(return_value=fake_return) + assert_equal(self.sync.recv_bytes(fake_arg, fake_kwarg=fake_kwarg), fake_return) + self.pipe.recv_bytes.assert_called_once_with(fake_arg, fake_kwarg=fake_kwarg) + self.sync.lock.__enter__.assert_called_once_with() + self.sync.lock.__exit__.assert_called_once_with(None, None, None) + + +class TronstorePoolTestCase(TestCase): @setup_teardown - def setup_thread_starter(self): - tronstore.is_shutdown = False - with mock.patch.object(tronstore.time, 'sleep') as self.sleep_patch: + def setup_tronstore_pool(self): + self.factory = mock.Mock() + self.pipe = mock.Mock() + self.store = mock.Mock() + with mock.patch('tron.serialize.runstate.tronstore.tronstore.Thread', autospec=True) \ + as self.thread_patch: + self.pool = tronstore.TronstorePool(self.factory, self.pipe, self.store) yield - def test_pool_size_limit(self): - fake_thread = mock.Mock() - fake_queue = Queue() - map(fake_queue.put, [fake_thread for i in range(tronstore.POOL_SIZE + 1)]) - fake_thread.is_alive = lambda: not self.sleep_patch.called - running_threads = [] - - def shutdown_tronstore(time): - tronstore.is_shutdown = True - self.sleep_patch.configure_mock(side_effect=shutdown_tronstore) - - tronstore.thread_starter(fake_queue, running_threads) - assert_equal(running_threads, []) - assert_equal(fake_thread.start.call_count, tronstore.POOL_SIZE+1) - self.sleep_patch.assert_called_once_with(0.5) - assert fake_queue.empty() - tronstore.is_shutdown = False # to make sure nothing weird happens - - def test_shutdown_condition(self): - fake_queue = Queue() - running_threads = [] - tronstore.is_shutdown = True - - with mock.patch.object(tronstore, '_remove_finished_threads', - return_value=0) as remove_patch: - tronstore.thread_starter(fake_queue, running_threads) - assert not remove_patch.called - assert not self.sleep_patch.called - assert_equal(running_threads, []) - tronstore.is_shutdown = False - - def test_running_thread_operations(self): - fake_queue = Queue() - fake_thread = mock.Mock() - running_threads = mock.Mock(__len__=lambda i: 0) - - def shutdown_tronstore(time): - tronstore.is_shutdown = True - running_threads.append = mock.Mock(side_effect=shutdown_tronstore) - - tronstore.is_shutdown = False - fake_queue.put(fake_thread) - - with mock.patch.object(tronstore, '_remove_finished_threads', - return_value=0) as remove_patch: - tronstore.thread_starter(fake_queue, running_threads) - remove_patch.assert_called_once_with(running_threads) - fake_thread.start.assert_called_once_with() - running_threads.append.assert_called_once_with(fake_thread) - - def test_remove_finished_threads(self): - fake_thread = mock.Mock(is_alive=mock.Mock(return_value=False)) - fake_get = mock.Mock(return_value=fake_thread) - fake_pop = mock.Mock() - fake_len = 5 - running_threads = mock.Mock( - __len__=lambda i: fake_len, - __getitem__=fake_get, - pop=fake_pop) - assert_equal(tronstore._remove_finished_threads(running_threads), 5) - calls = [mock.call(i) for i in range(fake_len-1, -1, -1)] - fake_get.assert_has_calls(calls) - fake_pop.assert_has_calls(calls) - assert_equal(fake_thread.is_alive.call_count, 5) + def test__init__(self): + assert isinstance(self.pool.request_queue, tronstore.Queue) + assert_equal(self.pool.response_factory, self.factory) + assert_equal(self.pool.pipe, self.pipe) + assert_equal(self.pool.store_class, self.store) + assert isinstance(self.pool.keep_working, tronstore.PoolBool) + assert self.pool.keep_working.value + assert_equal(self.pool.thread_pool, [self.thread_patch.return_value for i in range(self.pool.POOL_SIZE)]) + self.thread_patch.assert_any_call(target=tronstore.handle_requests, + args=( + self.pool.request_queue, + self.factory, + self.pipe, + self.store, + self.pool.keep_working + )) + + def test_start(self): + self.pool.keep_working.set(False) + self.pool.start() + assert self.pool.keep_working.value + assert not self.thread_patch.return_value.daemon + assert_equal(self.thread_patch.return_value.start.call_count, self.pool.POOL_SIZE) + + def test_stop(self): + self.pool.keep_working.set(True) + self.thread_patch.return_value.is_alive.return_value = False + with mock.patch.object(self.pool, 'has_work', return_value=False) \ + as work_patch: + self.pool.stop() + assert not self.pool.keep_working.value + work_patch.assert_called_once_with() + assert_equal(self.thread_patch.return_value.is_alive.call_count, self.pool.POOL_SIZE) + + def test_enqueue_work(self): + fake_work = 'youre_fired' + with mock.patch.object(self.pool.request_queue, 'put') as put_patch: + self.pool.enqueue_work(fake_work) + put_patch.assert_called_once_with(fake_work) + + def test_has_work(self): + with mock.patch.object(self.pool.request_queue, 'empty', return_value=True) \ + as empty_patch: + assert not self.pool.has_work() + empty_patch.assert_called_once_with() + + +if __name__ == "__main__": + run() diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 2b08d85dc..6e226f3a4 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -271,14 +271,12 @@ def update_from_config(self, state_config): return False if state_config.store_type not in schema.StatePersistenceTypes: - raise PersistenceStoreError("Unknown store type: %s" % store_type) + raise PersistenceStoreError("Unknown store type: %s" % state_config.store_type) if state_config.db_store_method not in schema.StateSerializationTypes \ - and store_type in ('sql', 'mongo'): - raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) + and state_config.store_type in ('sql', 'mongo'): + raise PersistenceStoreError("Unknown db store method: %s" % state_config.db_store_method) - # if self.state_manager is NullStateManager: - # self.state_manager = PersistenceManagerFactory.from_config(state_config) if not self.state_manager.update_from_config(state_config): return False else: diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 3e2475fce..5d6ba30e1 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -279,7 +279,8 @@ def restore(self, *args, **kwargs): return self.store.restore(*args, **kwargs) def cleanup(self): - self.store.cleanup() + with self.lock: + self.store.cleanup() def __repr__(self): return "SyncStore('%s')" % self.store.__repr__() From 19d2ff4a730bb17a7dbf9ac6ea6be73dbe62ddf4 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 30 Jul 2013 17:06:28 -0700 Subject: [PATCH 23/48] deleted chunking.py --- tron/serialize/runstate/tronstore/chunking.py | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 tron/serialize/runstate/tronstore/chunking.py diff --git a/tron/serialize/runstate/tronstore/chunking.py b/tron/serialize/runstate/tronstore/chunking.py deleted file mode 100644 index 0abe78c2d..000000000 --- a/tron/serialize/runstate/tronstore/chunking.py +++ /dev/null @@ -1,30 +0,0 @@ -CHUNK_SIGNING_STR = '\xDE\xAD\xBE\xEF\x00' - -class StoreChunkHandler(object): - """A simple chunk handler for dealing with string based I/O stream - messaging. Works by one end using the sign() function to sign the - serialized string and then using handle() on the opposite end of the wire. - - This is used by tronstore, as the pipes can get muddled.""" - - def __init__(self): - self.chunk = '' - - def sign(self, data): - """Sign a string to be sent.""" - return (data + CHUNK_SIGNING_STR) - - def handle(self, data): - """Handle a signed string, returning all individual strings that were - signed as a list. - """ - self.chunk += data - chunks = self.chunk.split(CHUNK_SIGNING_STR) - # split actually has this nice behavior where it makes the last element - # of the returned list an empty string if the original string - # ended with the sequence that was used to split with. This allows - # a nice, simple way to either get any remaining characters, or - # simply setting the chunk to '' again, without any extra code. - self.chunk = chunks[-1] if chunks else '' - chunks = chunks[:-1] - return chunks From 2a43c28b59d11bd55056c96213b22045deaf1071 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 30 Jul 2013 17:24:37 -0700 Subject: [PATCH 24/48] minor cleanup --- tools/migration/migrate_state_from_0.6.1_to_0.6.2.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index 6b1df5b37..befe60aed 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -69,9 +69,6 @@ def parse_options(): parser.add_option("-m", type="string", help="Set new state storing mechanism (store_type)", dest="store_method", default=None) - # parser.add_option("-t", type="string", - # help="Set new transport method", - # dest="transport_method", default=None) parser.add_option("-d", type="string", help="Set new SQL db serialization method (db_store_method)", dest="db_store_method", default=None) @@ -113,9 +110,6 @@ def compile_new_info(options, state_info, new_file): if options.store_method: new_state_info = new_state_info._replace(store_method=options.store_method) - # if options.transport_method: - # new_state_info = new_state_info._replace(transport_method=options.transport_method) - if options.db_store_method: new_state_info = new_state_info._replace(db_store_method=options.db_store_method) From 37015b4af5be63fd8561b1ff8556b5a00fd99bbf Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 14:10:38 -0700 Subject: [PATCH 25/48] updated conversion script --- tools/migration/migrate_state_from_0.6.1_to_0.6.2.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index befe60aed..db4307070 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -40,7 +40,7 @@ SQLAlchemy is the storing mechanism. Options for str are pickle, yaml, msgpack, and json. - -f str Set the path for the configuration file to str. This defaults to + -f str Set the path for the configuration dir to str. This defaults to /config """ @@ -68,7 +68,7 @@ def parse_options(): dest="new_connection_details", default=None) parser.add_option("-m", type="string", help="Set new state storing mechanism (store_type)", - dest="store_method", default=None) + dest="store_type", default=None) parser.add_option("-d", type="string", help="Set new SQL db serialization method (db_store_method)", dest="db_store_method", default=None) @@ -107,8 +107,8 @@ def compile_new_info(options, state_info, new_file): new_state_info = new_state_info._replace(name=new_file) - if options.store_method: - new_state_info = new_state_info._replace(store_method=options.store_method) + if options.store_type: + new_state_info = new_state_info._replace(store_type=options.store_type) if options.db_store_method: new_state_info = new_state_info._replace(db_store_method=options.db_store_method) @@ -187,7 +187,9 @@ def main(): old_store = get_old_state_store(state_info) print('Setting up the new state storing object...') new_state_info = compile_new_info(options, state_info, new_fname) - new_store = ParallelStore(new_state_info) + new_store = ParallelStore() + if not new_store.load_config(new_state_info): + raise AssertionError("Invalid configuration.") print('Copying metadata...') copy_metadata(old_store, new_store) From 0f2131d1c0ec5add45525bcf485c5ecd9002108d Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 14:34:34 -0700 Subject: [PATCH 26/48] another update to conversion script, fix for list stored versions --- tools/migration/migrate_state_from_0.6.1_to_0.6.2.py | 2 ++ tron/serialize/runstate/statemanager.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index db4307070..aec131d3b 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -139,6 +139,8 @@ def copy_metadata(old_store, new_store): meta_key_old = old_store.build_key(runstate.MCP_STATE, StateMetadata.name) old_metadata_dict = old_store.restore([meta_key_old]) if old_metadata_dict: + if 'version' in old_metadata_dict: + old_metadata_dict['version'] = (0, 6, 0, 2) old_metadata = old_metadata_dict[meta_key_old] meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 6e226f3a4..5ee23204f 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -73,6 +73,11 @@ def validate_metadata(cls, metadata): return version = metadata['version'] + if not isinstance(version, tuple): + try: + version = tuple(version) + except: + raise PersistenceStoreError("Stored metadata looks corrupted.") # Names (and state keys) changed in 0.5.2, requires migration # see tools/migration/migrate_state_to_namespace if version > cls.version or version < (0, 5, 2): From 837b4ed906618bb86302cc8799ae2c9fb67e3c20 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 14:36:16 -0700 Subject: [PATCH 27/48] comment cleanup --- tron/serialize/runstate/statemanager.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 5ee23204f..096014264 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -22,31 +22,6 @@ class VersionMismatchError(ValueError): class PersistenceStoreError(ValueError): """Raised if the store can not be created or fails a read or write.""" - -# class PersistenceManagerFactory(object): -# """Create a PersistentStateManager.""" - -# @classmethod -# def from_config(cls, persistence_config): -# store_type = persistence_config.store_type -# # transport_method = persistence_config.transport_method -# db_store_method = persistence_config.db_store_method -# buffer_size = persistence_config.buffer_size - -# if store_type not in schema.StatePersistenceTypes: -# raise PersistenceStoreError("Unknown store type: %s" % store_type) - -# # if transport_method not in schema.StateTransportTypes: -# # raise PersistenceStoreError("Unknown transport type: %s" % transport_method) - -# if db_store_method not in schema.StateSerializationTypes and store_type in ('sql', 'mongo'): -# raise PersistenceStoreError("Unknown db store method: %s" % db_store_method) - -# store = ParallelStore(persistence_config) -# buffer = StateSaveBuffer(buffer_size) -# return PersistentStateManager(store, buffer) - - class StateMetadata(object): """A data object for saving state metadata. Conforms to the same RunState interface as Jobs and Services. From e9b94cb4460801f6da05b08a7031bc2af5f682d7 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 14:43:40 -0700 Subject: [PATCH 28/48] updated version info --- tools/migration/migrate_state_from_0.6.1_to_0.6.2.py | 2 +- tron/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index aec131d3b..25d970e27 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -140,7 +140,7 @@ def copy_metadata(old_store, new_store): old_metadata_dict = old_store.restore([meta_key_old]) if old_metadata_dict: if 'version' in old_metadata_dict: - old_metadata_dict['version'] = (0, 6, 0, 2) + old_metadata_dict['version'] = (0, 6, 2, 0) old_metadata = old_metadata_dict[meta_key_old] meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) diff --git a/tron/__init__.py b/tron/__init__.py index bf67cb197..e161cf4c1 100644 --- a/tron/__init__.py +++ b/tron/__init__.py @@ -1,4 +1,4 @@ -__version_info__ = (0, 6, 1, 1) +__version_info__ = (0, 6, 2, 0) __version__ = ".".join("%s" % v for v in __version_info__) __author__ = 'Yelp ' From 1c185d93340ef62ae875b60df11c06e3909b5718 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 16:52:28 -0700 Subject: [PATCH 29/48] slight improvements to conversion, investigating serialization methods for dbs --- tools/migration/migrate_state_from_0.6.1_to_0.6.2.py | 10 +++++++--- tron/serialize/runstate/tronstore/serialize.py | 4 ++-- tron/serialize/runstate/tronstore/store.py | 6 +++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index 25d970e27..b49f9e759 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -139,12 +139,16 @@ def copy_metadata(old_store, new_store): meta_key_old = old_store.build_key(runstate.MCP_STATE, StateMetadata.name) old_metadata_dict = old_store.restore([meta_key_old]) if old_metadata_dict: - if 'version' in old_metadata_dict: - old_metadata_dict['version'] = (0, 6, 2, 0) old_metadata = old_metadata_dict[meta_key_old] + if 'version' in old_metadata: + old_metadata['version'] = (0, 6, 2, 0) meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) - assert_copied(new_store, old_metadata, meta_key_new) + try: + assert_copied(new_store, old_metadata, meta_key_new) + except AssertionError: + old_metadata['version'] = [0, 6, 2, 0] + assert_copied(new_store, old_metadata, meta_key_new) def copy_services(old_store, new_store, service_names): for service in service_names: diff --git a/tron/serialize/runstate/tronstore/serialize.py b/tron/serialize/runstate/tronstore/serialize.py index de38ae293..3d5cb3e2f 100644 --- a/tron/serialize/runstate/tronstore/serialize.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -33,7 +33,7 @@ def __str__(self): class JSONSerializer(object): @classmethod def serialize(cls, data): - return json.dumps(data, tuple_as_array=False) + return json.dumps(data) @classmethod def deserialize(cls, data_str): @@ -61,7 +61,7 @@ def serialize(cls, data): def deserialize(cls, data_str): if no_msgpack: raise SerializerModuleError('MessagePack not installed.') - return msgpack.unpackb(data_str, use_list=False) + return msgpack.unpackb(data_str) class YamlSerializer(object): diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 5d6ba30e1..e00c2eba1 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -193,7 +193,11 @@ class YamlStore(object): version of this (yamlstore.py), since key/value pairs are now saved INDIVIDUALLY rather than in batches, meaning saves are SLOOOOOOOW. - Seriously, you probably shouldn't use this unless you're doing something + How slow, you ask? Converting a standard Shelve store from 0.6.1 into + this object with test_config.yaml (and service_0 enabled) took about 4 + minutes. Going to a Shelve object instead took less than 5 seconds. + + Seriously, you shouldn't use this unless you're doing something really trivial and/or want a readable Yaml file. """ From 9e657de85a2f6464dc325e66515e030695236275 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 16:54:28 -0700 Subject: [PATCH 30/48] fix to sqlstore to use unicode in db --- tron/serialize/runstate/tronstore/store.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index e00c2eba1..8650f25f0 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -58,7 +58,10 @@ def __init__(self, name, connection_details, serializer): self.name = name self._connection = None self.serializer = serializer - self.engine = sql.create_engine(connection_details) + self.engine = sql.create_engine(connection_details, + connect_args={'check_same_thread': False}, + poolclass=sql.pool.StaticPool) + # self.engine.raw_connection().connection.text_factory = str self._setup_tables() def _setup_tables(self): @@ -101,7 +104,7 @@ def save(self, key, state_data, data_type): table = self._get_table(data_type) if table is None: return False - state_data = self.serializer.serialize(state_data) + state_data = unicode(repr(self.serializer.serialize(state_data))) update_result = conn.execute( table.update() .where(table.c.key == key) @@ -121,7 +124,7 @@ def restore(self, key, data_type): [table.c.state_data], table.c.key == key) ).fetchone() - return (True, self.serializer.deserialize(result[0])) if result else (False, None) + return (True, self.serializer.deserialize(eval(str(result[0])))) if result else (False, None) def cleanup(self): if self._connection: From 73e91776b135b5a9f7d061650b3fccc60e8c2851 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 5 Aug 2013 18:05:06 -0700 Subject: [PATCH 31/48] verification fixes to conversion script --- .../migrate_state_from_0.6.1_to_0.6.2.py | 26 ++++++++++++++----- tron/serialize/runstate/tronstore/store.py | 1 - 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index b49f9e759..ccb4b4c8b 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -47,6 +47,7 @@ import sys import os import copy +import simplejson as json from tron.commands import cmd_utils from tron.config import ConfigError @@ -127,12 +128,27 @@ def assert_copied(new_store, data, key): tronstore will serve the restore request BEFORE the save request, which will result in an Exception. We simply retry 5 times (which should be more than enough time for tronstore to serve the save request).""" + + if new_store.process.config.store_type == 'mongo': + data['_id'] = key.key for i in range(5): try: - assert data == new_store.restore([key])[key] - return - except Exception, e: + new_data = new_store.restore([key])[key] + except: continue + + if data == new_data: + return + + try: + if json.loads(json.dumps(data)) == new_data: + return + except TypeError: + # This should somehow replace the datetime object and check again + # via JSON if the objects are equal. + return + + import ipdb; ipdb.set_trace() raise AssertionError('The value %s failed to copy.' % key.iden) def copy_metadata(old_store, new_store): @@ -144,10 +160,6 @@ def copy_metadata(old_store, new_store): old_metadata['version'] = (0, 6, 2, 0) meta_key_new = new_store.build_key(runstate.MCP_STATE, StateMetadata.name) new_store.save([(meta_key_new, old_metadata)]) - try: - assert_copied(new_store, old_metadata, meta_key_new) - except AssertionError: - old_metadata['version'] = [0, 6, 2, 0] assert_copied(new_store, old_metadata, meta_key_new) def copy_services(old_store, new_store, service_names): diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 8650f25f0..067fa4869 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -61,7 +61,6 @@ def __init__(self, name, connection_details, serializer): self.engine = sql.create_engine(connection_details, connect_args={'check_same_thread': False}, poolclass=sql.pool.StaticPool) - # self.engine.raw_connection().connection.text_factory = str self._setup_tables() def _setup_tables(self): From 362bab75809194bd1df4ea9c996ea560a64ee945 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 6 Aug 2013 13:57:51 -0700 Subject: [PATCH 32/48] fix to reconfig bug --- tron/mcp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tron/mcp.py b/tron/mcp.py index 5b26e79bf..3aa1bf2a4 100644 --- a/tron/mcp.py +++ b/tron/mcp.py @@ -112,7 +112,8 @@ def update_state_watcher_config(self, state_config): """ if self.state_watcher.update_from_config(state_config): for job_container in self.jobs: - self.state_watcher.save_job_run(job_container.get_job_runs()) + for job_run in job_container.get_job_runs(): + self.state_watcher.save_job_run(job_run) self.state_watcher.save_job(job_container.get_job_state()) for service in self.services: self.state_watcher.save_service(service) From 2ed832b63526480c4e63613e915d6241384587c6 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 6 Aug 2013 17:36:25 -0700 Subject: [PATCH 33/48] docstring update --- tron/serialize/runstate/tronstore/tronstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index ce3334e8f..db11764fe 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -65,7 +65,7 @@ def handle_requests(request_queue, resp_factory, pipe, store_class, do_work): class SyncPipe(object): """An object to handle synchronization over pipe operations. In particular, - the send and recv functions should have mutexes as they are subject to + the send and recv functions have a mutex as they are subject to race conditions. """ From 42ebf728d4b8b5d4df4d3a3a29f430800a1acabe Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 7 Aug 2013 16:28:43 -0700 Subject: [PATCH 34/48] fixed broken test, code cleanup --- .../runstate/tronstore/store_test.py | 6 ++-- tron/serialize/runstate/tronstore/messages.py | 32 ------------------- 2 files changed, 3 insertions(+), 35 deletions(-) diff --git a/tests/serialize/runstate/tronstore/store_test.py b/tests/serialize/runstate/tronstore/store_test.py index 159fd3c05..c65ebdb2f 100644 --- a/tests/serialize/runstate/tronstore/store_test.py +++ b/tests/serialize/runstate/tronstore/store_test.py @@ -5,7 +5,7 @@ import contextlib from testify import TestCase, run, setup, assert_equal, teardown from tron.serialize.runstate.tronstore.store import ShelveStore, SQLStore, MongoStore, YamlStore, SyncStore, NullStore -from tron.serialize.runstate.tronstore.transport import JSONTransport +from tron.serialize.runstate.tronstore.serialize import cPickleSerializer from tron.serialize import runstate @@ -60,7 +60,7 @@ class SQLStoreTestCase(TestCase): @setup def setup_store(self): details = 'sqlite:///:memory:' - self.store = SQLStore('name', details, JSONTransport) + self.store = SQLStore('name', details, cPickleSerializer) @teardown def teardown_store(self): @@ -82,7 +82,7 @@ def test_save(self): self.store.save(key, state_data, data_type) rows = self.store.engine.execute(self.store.service_table.select()) - assert_equal(rows.fetchone(), ('dotes', self.store.serializer.serialize(state_data))) + assert_equal(rows.fetchone(), (u'dotes', unicode(repr(self.store.serializer.serialize(state_data))))) def test_restore_success(self): data_type = runstate.JOB_STATE diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 49257f23f..85d8b4e9d 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -1,7 +1,4 @@ -# from tron.serialize.runstate.tronstore.transport import JSONTransport from tron.serialize.runstate.tronstore.serialize import cPickleSerializer -# from tron.serialize.runstate.tronstore.transport import MsgPackTransport -# from tron.serialize.runstate.tronstore.transport import YamlTransport # a simple max integer to prevent ids from growing indefinitely MAX_MSG_ID = 2**32 - 1 @@ -28,10 +25,6 @@ def build(self, req_type, data_type, data): def from_msg(self, msg): return StoreRequest.from_message(self.serializer.deserialize(msg), self.serializer) - # def update_method(self, new_method): - # """Update the method used for message serialization.""" - # self.serializer = transport_class_map[new_method] - def get_method(self): return self.serializer @@ -53,10 +46,6 @@ def build(self, success, req_id, data): def from_msg(self, msg): return StoreResponse.from_message(self.serializer.deserialize(msg), self.serializer) - # def update_method(self, new_method): - # """Update the method used for message serialization.""" - # self.serializer = transport_class_map[new_method] - def get_method(self): return self.serializer @@ -77,7 +66,6 @@ def __init__(self, req_id, req_type, data_type, data, method): self.data = data self.data_type = data_type self.method = method - # self.serialized = self.get_serialized() @classmethod def from_message(cls, msg_data, method): @@ -92,18 +80,6 @@ def serialized(self): self.data_type, self.data)) - # def update_method(self, new_method): - # """Update the method used for message serialization.""" - # self.method = transport_class_map['new_method'] - # self.serialized = self.get_serialized() - - # def get_serialized(self): - # return self.method.serialize(( - # self.id, - # self.req_type, - # self.data_type, - # self.data)) - class StoreResponse(object): """An object representing a response from tronstore. The response has three @@ -128,11 +104,3 @@ def from_message(cls, msg_data, method): @property def serialized(self): return self.method.serialize((self.id, self.success, self.data)) - - # def update_method(self, new_method): - # """Update the method used for message serialization.""" - # self.method = transport_class_map['new_method'] - # self.serialized = self.get_serialized() - - # def get_serialized(self): - # return self.method.serialize((self.id, self.success, self.data)) From 82b0eebc1e9adeb449167095d9716f336b6d9a66 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 7 Aug 2013 16:33:30 -0700 Subject: [PATCH 35/48] cleaned up commented code --- tron/serialize/runstate/statemanager.py | 4 ---- tron/serialize/runstate/tronstore/messages.py | 1 - tron/serialize/runstate/tronstore/parallelstore.py | 2 -- tron/serialize/runstate/tronstore/process.py | 2 -- tron/serialize/runstate/tronstore/tronstore.py | 1 - 5 files changed, 10 deletions(-) diff --git a/tron/serialize/runstate/statemanager.py b/tron/serialize/runstate/statemanager.py index 096014264..e503faece 100644 --- a/tron/serialize/runstate/statemanager.py +++ b/tron/serialize/runstate/statemanager.py @@ -6,10 +6,6 @@ from tron.config import schema from tron.core import job, jobrun, service from tron.serialize import runstate -# from tron.serialize.runstate.mongostore import MongoStateStore -# from tron.serialize.runstate.shelvestore import ShelveStateStore -# from tron.serialize.runstate.sqlalchemystore import SQLAlchemyStateStore -# from tron.serialize.runstate.yamlstore import YamlStateStore from tron.serialize.runstate.tronstore.parallelstore import ParallelStore from tron.utils import observer diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index 85d8b4e9d..a7138941b 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -94,7 +94,6 @@ def __init__(self, req_id, success, data, method): self.success = success self.data = data self.method = method - # self.serialized = self.get_serialized() @classmethod def from_message(cls, msg_data, method): diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index ebddbe8f9..0b0268754 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -72,8 +72,6 @@ def load_config(self, new_config): response = self.process.send_request_get_response(config_req) if response.success: self.process.update_config(new_config) - # self.request_factory.update_method(new_config.transport_method) - # self.response_factory.update_method(new_config.transport_method) return True else: return False diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index a082c7aa2..0e3efe425 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -82,8 +82,6 @@ def _poll_for_response(self, id, timeout): be fine. """ if id in self.orphaned_responses: - # response = self.orphaned_responses[id] - # del self.orphaned_responses[id] return self.orphaned_responses.pop(id) while self.pipe.poll(timeout): diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index db11764fe..95b4d0de7 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -50,7 +50,6 @@ def handle_requests(request_queue, resp_factory, pipe, store_class, do_work): if request.req_type == msg_enums.REQUEST_SAVE: store_class.save(request.data[0], request.data[1], request.data_type) - # pipe.send_bytes(resp_factory.build(success, request.id, '').serialized) elif request.req_type == msg_enums.REQUEST_RESTORE: success, data = store_class.restore(request.data, request.data_type) From 449097f086eb6750705e01586f14ee8cf39c5dba Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Wed, 7 Aug 2013 17:37:21 -0700 Subject: [PATCH 36/48] fixed JobState.status to return correctly if a run is starting --- tron/core/job.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tron/core/job.py b/tron/core/job.py index 30c1d9ece..5ee920048 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -58,7 +58,9 @@ def status(self, job_runs): """Current status of the job. Takes a JobRunCollection as an argument.""" if not self.enabled: return self.STATUS_DISABLED - if job_runs.get_run_by_state(ActionRun.STATE_RUNNING): + + if (job_runs.get_run_by_state(ActionRun.STATE_RUNNING) or + job_runs.get_run_by_state(ActionRun.STATE_STARTING)): return self.STATUS_RUNNING if (job_runs.get_run_by_state(ActionRun.STATE_SCHEDULED) or From d2f3ab5fbf314f82bc2af9d2395fde432a95850e Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 8 Aug 2013 14:03:41 -0700 Subject: [PATCH 37/48] fixed serializers, migrate_state --- tools/migration/migrate_state.py | 11 ++++++-- .../migrate_state_from_0.6.1_to_0.6.2.py | 14 ++++------ .../runstate/tronstore/parallelstore.py | 3 +- .../serialize/runstate/tronstore/serialize.py | 28 ++++++++++++++++--- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/tools/migration/migrate_state.py b/tools/migration/migrate_state.py index 4be21e808..1d0045125 100644 --- a/tools/migration/migrate_state.py +++ b/tools/migration/migrate_state.py @@ -17,7 +17,7 @@ import optparse from tron.config import manager, schema from tron.serialize import runstate -from tron.serialize.runstate.statemanager import PersistenceManagerFactory +from tron.serialize.runstate.statemanager import PersistentStateManager from tron.utils import tool_utils @@ -49,11 +49,16 @@ def parse_options(): def get_state_manager_from_config(config_path, working_dir): """Return a state manager from the configuration. """ + if not working_dir: + working_dir = config_path config_manager = manager.ConfigManager(config_path) config_container = config_manager.load() state_config = config_container.get_master().state_persistence with tool_utils.working_dir(working_dir): - return PersistenceManagerFactory.from_config(state_config) + ret = PersistentStateManager() + if not ret.update_from_config(state_config): + raise SystemError("%s failed to load." % config_path) + return ret def get_current_config(config_path): @@ -103,6 +108,8 @@ def convert_state(opts): dest_manager.cleanup() + print "Hang on, saving everything to the destination object..." + if __name__ == "__main__": opts, _args = parse_options() diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index ccb4b4c8b..4a89ea461 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -60,6 +60,7 @@ from tron.serialize.runstate.sqlalchemystore import SQLAlchemyStateStore from tron.serialize.runstate.tronstore.parallelstore import ParallelStore from tron.serialize.runstate.statemanager import StateMetadata +from tron.serialize.runstate.tronstore.serialize import MsgPackSerializer def parse_options(): usage = "usage: %prog [options] " @@ -126,12 +127,12 @@ def compile_new_info(options, state_info, new_file): def assert_copied(new_store, data, key): """A small function to counter race conditions. It's possible that tronstore will serve the restore request BEFORE the save request, which - will result in an Exception. We simply retry 5 times (which should be more + will result in an Exception. We simply retry 10 times (which should be more than enough time for tronstore to serve the save request).""" if new_store.process.config.store_type == 'mongo': data['_id'] = key.key - for i in range(5): + for i in range(10): try: new_data = new_store.restore([key])[key] except: @@ -141,14 +142,11 @@ def assert_copied(new_store, data, key): return try: - if json.loads(json.dumps(data)) == new_data: + if MsgPackSerializer.deserialize(MsgPackSerializer.serialize(data)) == new_data: return - except TypeError: - # This should somehow replace the datetime object and check again - # via JSON if the objects are equal. - return + except: + continue - import ipdb; ipdb.set_trace() raise AssertionError('The value %s failed to copy.' % key.iden) def copy_metadata(old_store, new_store): diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index 0b0268754..f97a0422c 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -77,4 +77,5 @@ def load_config(self, new_config): return False def __repr__(self): - return "ParallelStore" + store = self.process.config.store_type if self.process.config else None + return "ParallelStore(%s)" % store diff --git a/tron/serialize/runstate/tronstore/serialize.py b/tron/serialize/runstate/tronstore/serialize.py index 3d5cb3e2f..784517715 100644 --- a/tron/serialize/runstate/tronstore/serialize.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -5,6 +5,7 @@ This is also used by the SQLAlchemy store object, an option for saving state with tronstore, by serializing the state data into a string that's saved in a SQL database, or by deserializing strings that are saved into state data.""" +import datetime import simplejson as json import cPickle as pickle @@ -21,6 +22,25 @@ no_yaml = True +def custom_decode(obj): + try: + if b'__tuple__' in obj: + return tuple(custom_decode(o) for o in obj['items']) + elif b'__datetime__' in obj: + obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") + return obj + except: + return obj + + +def custom_encode(obj): + if isinstance(obj, tuple): + return {'__tuple__': True, 'items': [custom_encode(e) for e in obj]} + elif isinstance(obj, datetime.datetime): + return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} + return obj + + class SerializerModuleError(Exception): """Raised if a serialization module is used without it being installed.""" def __init__(self, code): @@ -33,11 +53,11 @@ def __str__(self): class JSONSerializer(object): @classmethod def serialize(cls, data): - return json.dumps(data) + return json.dumps(data, default=custom_encode, tuple_as_array=False) @classmethod def deserialize(cls, data_str): - return json.loads(data_str) + return json.loads(data_str, object_hook=custom_decode) class cPickleSerializer(object): @@ -55,13 +75,13 @@ class MsgPackSerializer(object): def serialize(cls, data): if no_msgpack: raise SerializerModuleError('MessagePack not installed.') - return msgpack.packb(data) + return msgpack.packb(data, default=custom_encode) @classmethod def deserialize(cls, data_str): if no_msgpack: raise SerializerModuleError('MessagePack not installed.') - return msgpack.unpackb(data_str) + return msgpack.unpackb(data_str, object_hook=custom_decode, use_list=0) class YamlSerializer(object): From a5583c676e8e2e60f856382562620fe6488cb30e Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 8 Aug 2013 15:10:07 -0700 Subject: [PATCH 38/48] all the docstrings --- .../migrate_state_from_0.6.1_to_0.6.2.py | 42 ++++++++++--------- tron/serialize/runstate/tronstore/messages.py | 10 +++-- .../runstate/tronstore/parallelstore.py | 12 +++--- tron/serialize/runstate/tronstore/process.py | 38 +++++++++-------- .../serialize/runstate/tronstore/serialize.py | 5 +++ .../serialize/runstate/tronstore/tronstore.py | 26 ++++++------ 6 files changed, 72 insertions(+), 61 deletions(-) diff --git a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py index 4a89ea461..403d7ff12 100644 --- a/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py +++ b/tools/migration/migrate_state_from_0.6.1_to_0.6.2.py @@ -2,27 +2,31 @@ This is a script to convert old state storing containers into the new objects used by Tron v0.6.2 and tronstore. The script will use the same -mechanism for storing state as specified in the Tron configuration file -(except for SQL, which will default to using simplejson for serializing -state data into its database). This can be overriden via command line -options, which allow for full configuration of the mechanisms used to store -the new state objects. +mechanism for storing state as specified in the Tron configuration file. +Config elements can be overriden via command line options, which allows for +full configuration of the mechanism used to store the new state object. -Please upgrade to Tron v0.6.2 before running this script. Also note that -migrate_state.py will NOT work until running this script, as it has been -changed to work with v0.6.2's version of state storing. +Please ensure that you have Tron v0.6.2 before running this script. Also note +that migrate_state.py will NOT work again until running this script, as it has +been changed to work with v0.6.2's method of state storing. -Make sure that the working dir is the same as the one used in your Tron -configuration! Otherwise, this script won't be able to load the config file -and make the magic happen. +The working dir should generally be the same as the one used when launching +trond, but should contain the file pointed to by the configuration file. +The script attempts to load a configuration from /config by +default, or whatever -f was set to. ***IMPORTANT*** When using SQLAlchemy/MongoDB storing mechanisms, the -c option for setting -NEW connection detail parameters MUST be set. Because we don't want to clobber -the old data, this script requires that there is a new database for saving -the new state data, meaning new connection parameters. This script -doesn't check rigorously that the new connection details given are valid; -however, it does verify that the details aren't exactly the same. +connection detail parameters MUST be set. + +HOWEVER, THE SCRIPT DOES NOT CHECK WHETHER OR NOT THE CONNECTION DETAILS +ARE THE SAME, NOR IF YOU ARE GOING TO CLOBBER YOUR OLD DATABASE WITH THE GIVEN +CONNECTION AND CONFIGURATION PARAMETERS. + +Please especially ensure that you are not connecting to the exact +same SQL database that holds your old state_data, or you are likely to run +into a large number of strange problems and inconsistencies. +***IMPORTANT*** Command line options: @@ -47,7 +51,6 @@ import sys import os import copy -import simplejson as json from tron.commands import cmd_utils from tron.config import ConfigError @@ -115,11 +118,10 @@ def compile_new_info(options, state_info, new_file): if options.db_store_method: new_state_info = new_state_info._replace(db_store_method=options.db_store_method) - if options.new_connection_details \ - and options.new_connection_details != state_info.connection_details: + if options.new_connection_details: new_state_info = new_state_info._replace(connection_details=options.new_connection_details) elif new_state_info.store_type in ('sql', 'mongo'): - raise ConfigError('Must specify new connection_details using -c to use %s' + raise ConfigError('Must specify connection_details using -c to use %s' % new_state_info.store_type) return new_state_info diff --git a/tron/serialize/runstate/tronstore/messages.py b/tron/serialize/runstate/tronstore/messages.py index a7138941b..4e3be9bef 100644 --- a/tron/serialize/runstate/tronstore/messages.py +++ b/tron/serialize/runstate/tronstore/messages.py @@ -5,7 +5,11 @@ class StoreRequestFactory(object): - """A factory to generate requests by giving each a unique id.""" + """A factory to generate requests by giving each a unique id. + The serialization method should usually be cPickle. However, you can simply + change what serialization class is used in the __init__ method- just + make sure to change it in the StoreResponseFactory as well! + """ def __init__(self): self.serializer = cPickleSerializer @@ -31,9 +35,7 @@ def get_method(self): class StoreResponseFactory(object): """A factory to generate responses that need to be converted to serialized - strings and back. The factory itself just keeps track of what serialization - method was specified by the configuration, and then constructs specific - StoreResponse objects using that method. + strings and back.. """ def __init__(self): diff --git a/tron/serialize/runstate/tronstore/parallelstore.py b/tron/serialize/runstate/tronstore/parallelstore.py index f97a0422c..9e9ab65b1 100644 --- a/tron/serialize/runstate/tronstore/parallelstore.py +++ b/tron/serialize/runstate/tronstore/parallelstore.py @@ -32,12 +32,12 @@ def __hash__(self): class ParallelStore(object): - """Persist state using a paralleled storing mechanism, tronstore. This uses - the Twisted library to run the tronstore executable in a separate - process, and handles all communication between trond and tronstore. + """Persist state using a parallel storing mechanism, tronstore. This uses + the python mulitprocessing module to run the tronstore executable in + another process, and handles all communication between trond and tronstore. This class handles construction of all messages that need to be sent - to tronstore based on requests given by the MCP.""" + to tronstore based on whatever method was called.""" def __init__(self): self.request_factory = StoreRequestFactory() @@ -67,7 +67,9 @@ def cleanup(self): def load_config(self, new_config): """Reconfigure the storing mechanism to use a new configuration - by shutting down and restarting tronstore.""" + by shutting down and restarting tronstore. THIS MUST BE CALLED + AT LEAST ONCE, as tronstore is started with a null configuration + whenever a ParallelStore object is created.""" config_req = self.request_factory.build(msg_enums.REQUEST_CONFIG, '', new_config) response = self.process.send_request_get_response(config_req) if response.success: diff --git a/tron/serialize/runstate/tronstore/process.py b/tron/serialize/runstate/tronstore/process.py index 0e3efe425..cd21636c6 100644 --- a/tron/serialize/runstate/tronstore/process.py +++ b/tron/serialize/runstate/tronstore/process.py @@ -19,14 +19,21 @@ def __str__(self): class StoreProcessProtocol(object): - """The class that actually communicates with tronstore. This is a subclass - of the twisted ProcessProtocol class, which has a set of internals that can - communicate with a child proccess via stdin/stdout via interrupts. - - Because of this I/O structure imposed by twisted, there are two types of - messages: requests and responses. Responses are always of the same form, - while requests have an enumerator (see msg_enums.py) to identify the - type of request. + """The class that actually spawns and handles the tronstore process. + + This class uses the python multiprocessing module. Upon creation, it + starts tronstore with a null configuration. A reconfiguration request + must be sent to tronstore via one of the supplied object methods before + it will be able to actually perform saves and restores. Calling + update_config on this object will simply update the saved configuration + object- it won't actually update the configuration that the tronstore + process itself is using unless a _verify_is_alive fails and tronstore is + restarted. + + Communication with the process is handled by a Pipe object, which can + simply pass entire Python objects via Pickle. Despite this, we still + serialize all requests with cPickle before sending them, as cPickle + is much faster and effectively the same as cPickle. """ # This timeout MUST be longer than the POLL_TIMEOUT in tronstore! SHUTDOWN_TIMEOUT = 100.0 @@ -40,9 +47,7 @@ def __init__(self): self._start_process() def _start_process(self): - """Spawn the tronstore process. The arguments given to tronstore must - match the signature for tronstore.main. - """ + """Spawn the tronstore process with the saved configuration.""" self.pipe, child_pipe = Pipe() store_args = (self.config, child_pipe) @@ -52,7 +57,8 @@ def _start_process(self): def _verify_is_alive(self): """A check to verify that tronstore is alive. Attempts to restart - tronstore if it finds that it exited for some reason.""" + tronstore if it finds that it exited for some reason. + """ if not self.process.is_alive(): code = self.process.exitcode log.warn("tronstore exited prematurely with status code %d. Attempting to restart." % code) @@ -76,10 +82,6 @@ def _poll_for_response(self, id, timeout): any responses that it isn't looking for into a dict, and tries to retrieve a matching response from this dict before pulling new responses. - - If Tron is extended into a synchronous program, simply just add a - lock around this function ( with mutex.lock(): ) and everything'll - be fine. """ if id in self.orphaned_responses: return self.orphaned_responses.pop(id) @@ -114,8 +116,8 @@ def send_request_get_response(self, request): def send_request_shutdown(self, request): """Shut down the process protocol. Waits for SHUTDOWN_TIMEOUT seconds - for tronstore to send a response, after which it kills both pipes - and the process itself. + for tronstore to send a shutdown response, killing both pipes and the + process itself if no shutdown response was returned. Calling this prevents ANY further requests from being made to tronstore as the process will be killed. diff --git a/tron/serialize/runstate/tronstore/serialize.py b/tron/serialize/runstate/tronstore/serialize.py index 784517715..b01d43abd 100644 --- a/tron/serialize/runstate/tronstore/serialize.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -23,6 +23,10 @@ def custom_decode(obj): + """A custom decoder for datetime and tuple objects. + The tuple part only works for JSON, as MsgPack handles tuples and lists + itself no matter what. + """ try: if b'__tuple__' in obj: return tuple(custom_decode(o) for o in obj['items']) @@ -34,6 +38,7 @@ def custom_decode(obj): def custom_encode(obj): + """A custom encoder for datetime and tuple objects.""" if isinstance(obj, tuple): return {'__tuple__': True, 'items': [custom_encode(e) for e in obj]} elif isinstance(obj, datetime.datetime): diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 95b4d0de7..36573a6a4 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -1,18 +1,4 @@ #!/usr/bin/env python -"""This process is spawned by trond in order to offload state save/load -operations such that trond can focus on the more important things without -blocking for large chunks of time. It takes arguments in the main method -passed by python's multiprocessing module in order to configure itself and use -the correct methods for state saving and message transport with trond. - -Messages are sent via Pipes (also part of python's multiprocessing module). -This allows for easy polling and no need to handle chunking of messages. - -The process intercepts the two shutdown signals (SIGINT and SIGTERM) in order -to prevent the process from exiting early when trond wants to do some final -shutdown things (realistically, trond should be handling all shutdown -operations, as this is a child process.) -""" import time import signal import os @@ -270,6 +256,18 @@ def main(config, pipe): """The main method to start Tronstore with. Simply takes the configuration and pipe objects, and then registers some null signal handlers before passing everything off to TronstoreMain. + + This process is spawned by trond in order to offload state save/load + operations such that trond can focus on the more important things without + blocking for chunks of time. + + Messages are sent via Pipes (also part of python's multiprocessing module). + This allows for easy polling and no need to handle chunking of messages. + + The process intercepts the two shutdown signals (SIGINT and SIGTERM) in order + to prevent the process from exiting early when trond wants to do some final + shutdown things (realistically, trond should be handling all shutdown + operations, as this is a child process.) """ _register_null_handlers() From 87e6890e6255e60ea852daca266035de39e965e1 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Thu, 8 Aug 2013 15:30:12 -0700 Subject: [PATCH 39/48] reduced default POOL_SIZE --- tron/serialize/runstate/tronstore/tronstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index 36573a6a4..cc14f0537 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -97,7 +97,7 @@ class TronstorePool(object): workers, which send an appropriate response. """ - POOL_SIZE = 35 + POOL_SIZE = 16 def __init__(self, resp_fact, pipe, store): """Initialize the thread pool. Please make a new pool if any of the From 7143081aca7497f18e0154645d8d694ca47893ad Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Fri, 9 Aug 2013 14:46:11 -0700 Subject: [PATCH 40/48] improved sqlstore, fixed restore_state on queued runs --- tests/core/job_test.py | 29 +++++++++++++++++ .../runstate/tronstore/store_test.py | 6 ++-- tron/core/job.py | 15 ++++++--- .../serialize/runstate/tronstore/serialize.py | 8 +++++ tron/serialize/runstate/tronstore/store.py | 31 ++++++++++++++----- .../serialize/runstate/tronstore/tronstore.py | 2 +- 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/tests/core/job_test.py b/tests/core/job_test.py index e53290ec5..68c31b285 100644 --- a/tests/core/job_test.py +++ b/tests/core/job_test.py @@ -504,6 +504,35 @@ def mock_eventloop(self): def teardown_job(self): event.EventManager.reset() + def test_restore_state_scheduled(self): + mock_scheduled = [mock.Mock(), mock.Mock()] + with contextlib.nested( + mock.patch.object(self.job_scheduler.job_runs, 'get_scheduled', + return_value=iter(mock_scheduled)), + mock.patch.object(self.job_scheduler, 'schedule'), + mock.patch.object(self.job_scheduler, '_set_callback') + ) as (get_patch, sched_patch, back_patch): + self.job_scheduler.restore_state() + get_patch.assert_called_once_with() + calls = [mock.call(m) for m in mock_scheduled] + back_patch.assert_has_calls(calls) + sched_patch.assert_called_once_with() + + def test_restore_state_queued(self): + queued = mock.Mock() + with contextlib.nested( + mock.patch.object(self.job_scheduler.job_runs, 'get_scheduled', + return_value=iter([])), + mock.patch.object(self.job_scheduler.job_runs, 'get_first_queued', + return_value=queued), + mock.patch.object(self.job_scheduler, 'schedule'), + mock.patch.object(self.job_scheduler, '_set_callback') + ) as (get_patch, queue_patch, sched_patch, back_patch): + self.job_scheduler.restore_state() + get_patch.assert_called_once_with() + back_patch.assert_called_once_with(queued, run_queued=True) + sched_patch.assert_called_once_with() + def test_schedule(self): with mock.patch.object(self.job_scheduler.job_state, 'is_enabled', new=True): diff --git a/tests/serialize/runstate/tronstore/store_test.py b/tests/serialize/runstate/tronstore/store_test.py index c65ebdb2f..398fdcd7c 100644 --- a/tests/serialize/runstate/tronstore/store_test.py +++ b/tests/serialize/runstate/tronstore/store_test.py @@ -13,13 +13,13 @@ class ShelveStoreTestCase(TestCase): @setup def setup_store(self): - self.filename = os.path.join(tempfile.gettempdir(), 'state') + self.filename = os.path.join(tempfile.gettempdir(), 'tmp_shelve.state') self.store = ShelveStore(self.filename, None, None) @teardown def teardown_store(self): - os.unlink(self.filename) self.store.cleanup() + os.unlink(self.filename) def test__init__(self): assert_equal(self.filename, self.store.fname) @@ -82,7 +82,7 @@ def test_save(self): self.store.save(key, state_data, data_type) rows = self.store.engine.execute(self.store.service_table.select()) - assert_equal(rows.fetchone(), (u'dotes', unicode(repr(self.store.serializer.serialize(state_data))))) + assert_equal(rows.fetchone(), (u'dotes', unicode(repr(self.store.serializer.serialize(state_data))), u'pickle')) def test_restore_success(self): data_type = runstate.JOB_STATE diff --git a/tron/core/job.py b/tron/core/job.py index 5ee920048..b84498d5f 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -130,9 +130,14 @@ def __init__(self, job_runs, job_config, job_state, scheduler, actiongraph, def restore_state(self): """Restore the job state and schedule any JobRuns.""" - scheduled = self.job_runs.get_scheduled() - for job_run in scheduled: - self._set_callback(job_run) + scheduled = [run for run in self.job_runs.get_scheduled()] + if scheduled: + for job_run in scheduled: + self._set_callback(job_run) + else: + queued_run = self.job_runs.get_first_queued() + if queued_run: + self._set_callback(queued_run, run_queued=True) # Ensure we have at least 1 scheduled run self.schedule() @@ -176,11 +181,11 @@ def schedule(self): return self.create_and_schedule_runs() - def _set_callback(self, job_run): + def _set_callback(self, job_run, run_queued=False): """Set a callback for JobRun to fire at the appropriate time.""" log.info("Scheduling next Jobrun for %s", self.config.name) seconds = job_run.seconds_until_run_time() - eventloop.call_later(seconds, self.run_job, job_run) + eventloop.call_later(seconds, self.run_job, job_run, run_queued=run_queued) # TODO: new class for this method def run_job(self, job_run, run_queued=False): diff --git a/tron/serialize/runstate/tronstore/serialize.py b/tron/serialize/runstate/tronstore/serialize.py index b01d43abd..b7b39d260 100644 --- a/tron/serialize/runstate/tronstore/serialize.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -56,6 +56,8 @@ def __str__(self): class JSONSerializer(object): + name = 'json' + @classmethod def serialize(cls, data): return json.dumps(data, default=custom_encode, tuple_as_array=False) @@ -66,6 +68,8 @@ def deserialize(cls, data_str): class cPickleSerializer(object): + name = 'pickle' + @classmethod def serialize(cls, data): return pickle.dumps(data) @@ -76,6 +80,8 @@ def deserialize(cls, data_str): class MsgPackSerializer(object): + name = 'msgpack' + @classmethod def serialize(cls, data): if no_msgpack: @@ -90,6 +96,8 @@ def deserialize(cls, data_str): class YamlSerializer(object): + name = 'yaml' + @classmethod def serialize(cls, data): if no_yaml: diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 067fa4869..89b07bbb6 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -67,16 +67,20 @@ def _setup_tables(self): self._metadata = sql.MetaData() self.job_state_table = sql.Table('job_state_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text)) + sql.Column('state_data', sql.Text), + sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.service_table = sql.Table('service_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text)) + sql.Column('state_data', sql.Text), + sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.job_run_table = sql.Table('job_run_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text)) + sql.Column('state_data', sql.Text), + sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.metadata_table = sql.Table('metadata_table', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text)) + sql.Column('state_data', sql.Text), + sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self._metadata.create_all(self.engine) @@ -104,14 +108,17 @@ def save(self, key, state_data, data_type): if table is None: return False state_data = unicode(repr(self.serializer.serialize(state_data))) + serial_method = self.serializer.name update_result = conn.execute( table.update() .where(table.c.key == key) - .values(state_data=state_data)) + .values(state_data=state_data, + serial_method=serial_method)) if not update_result.rowcount: conn.execute( table.insert() - .values(key=key, state_data=state_data)) + .values(key=key, state_data=state_data, + serial_method=serial_method)) return True def restore(self, key, data_type): @@ -120,10 +127,18 @@ def restore(self, key, data_type): if table is None: return (False, None) result = conn.execute(sql.sql.select( - [table.c.state_data], + [table.c.state_data, table.c.serial_method], table.c.key == key) ).fetchone() - return (True, self.serializer.deserialize(eval(str(result[0])))) if result else (False, None) + if not result: + return (False, None) + elif str(result[1]) != self.serializer.name: + # TODO: If/when we have logging in the Tronstore process, + # log here that the db_store_method was different + serializer = serialize_class_map[str(result[1])] + return (True, serializer.deserialize(eval(str(result[0])))) + else: + return (True, self.serializer.deserialize(eval(str(result[0])))) def cleanup(self): if self._connection: diff --git a/tron/serialize/runstate/tronstore/tronstore.py b/tron/serialize/runstate/tronstore/tronstore.py index cc14f0537..ea0639eb6 100644 --- a/tron/serialize/runstate/tronstore/tronstore.py +++ b/tron/serialize/runstate/tronstore/tronstore.py @@ -146,7 +146,7 @@ class TronstoreMain(object): """ # this can be rather long- it's only real use it to clean up tronstore - # in case it's zombied... however, it should be SHORTER than + # in case it's orphaned... however, it should be SHORTER than # SHUTDOWN_TIMEOUT in process.py. in addition, making this longer # can cause trond to take longer to fully shutdown. POLL_TIMEOUT = 2.0 From 7c39f0824a2e1b4754302bbc9aa3b08c630b011c Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Fri, 9 Aug 2013 16:33:10 -0700 Subject: [PATCH 41/48] updated docs --- docs/config.rst | 13 +++++++++++++ docs/man_tronview.rst | 3 +++ tron/config/config_parse.py | 6 ------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 419bc2462..84226f00c 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -215,6 +215,18 @@ State Persistence The number of save calls to buffer before writing the state. Defaults to 1, which is no buffering. + **db_store_method** + The method to use for saving state information to a SQL database. Only used if store_type is sql. + + Valid options are: + **json** - uses the `simplejson` module. + + **msgpack** - uses the `msgpack` module, from the msgpack-python package (tested with version 0.3.0). + + **pickle** - uses the `cPickle` module. be careful with this one, as pickle is Turing complete. + + **yaml** - uses the `yaml` module, from the PyYaml package (tested with version 3.10). + Example:: @@ -223,6 +235,7 @@ Example:: name: local_sqlite connection_details: "sqlite:///dest_state.db" buffer_size: 1 # No buffer + db_store_method: json .. _action_runners: diff --git a/docs/man_tronview.rst b/docs/man_tronview.rst index 1957c7f6e..cf8367741 100644 --- a/docs/man_tronview.rst +++ b/docs/man_tronview.rst @@ -69,6 +69,9 @@ Options ``-s, --save`` Save server and color options to client config file (~/.tron) +``--namespace`` + Only show jobs and services from the specified namespace + States ---------- diff --git a/tron/config/config_parse.py b/tron/config/config_parse.py index 477083826..d8c9151a7 100644 --- a/tron/config/config_parse.py +++ b/tron/config/config_parse.py @@ -392,12 +392,6 @@ class ValidateStatePersistence(Validator): defaults = { 'buffer_size': 1, 'connection_details': None, - # This is a tricky one. MessagePack isn't a default python library, - # so it feels a bit wrong to make it a configuration default. However, - # Yaml's terrible and simplejson is slower than both msgpack and - # pickle. Pickle MIGHT be an okay default, but I'm nervous about - # defaulting people into using a Turing Complete serialization method - # for SQL storing. 'db_store_method': 'json', } From 98f7a2394db0ff1c343ce144b00dec309bf414f4 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 09:58:14 -0700 Subject: [PATCH 42/48] made job scheduler properly watch runs on restore --- tron/core/job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tron/core/job.py b/tron/core/job.py index b84498d5f..c5fd53fce 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -446,6 +446,7 @@ def restore_state(self, state_data): self.node_pool) for run in job_runs: self.watcher.watch(run) + self.job_scheduler.watch(run, jobrun.JobRun.NOTIFY_DONE) self.job_state.restore_state(job_state_data) self.job_scheduler.restore_state() self.job_state.set_run_ids(self.job_runs.get_run_numbers()) # consistency From d96de3fb0042c5622d591877957f79cf451ba6de Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 10:01:24 -0700 Subject: [PATCH 43/48] enforced new watch condition in unit test --- tests/core/job_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/job_test.py b/tests/core/job_test.py index 68c31b285..ea9d43677 100644 --- a/tests/core/job_test.py +++ b/tests/core/job_test.py @@ -112,6 +112,8 @@ def test_restore_state(self): assert not self.job.enabled calls = [mock.call(job_runs[i]) for i in xrange(len(job_runs))] self.job.watcher.watch.assert_has_calls(calls) + calls = [mock.call(job_runs[i], jobrun.JobRun.NOTIFY_DONE) for i in xrange(len(job_runs))] + self.job_scheduler.watch.assert_has_calls(calls) assert_equal(self.job.job_state.state_data, state_data[0]) self.job.job_runs.restore_state.assert_called_once_with( sorted(run_data, key=lambda data: data['run_num'], reverse=True), From c0d2d6f32c350c887497891e3169916f5d766131 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 11:21:07 -0700 Subject: [PATCH 44/48] style cleanup for queued run callback --- tests/core/job_test.py | 6 +++--- tron/core/job.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/core/job_test.py b/tests/core/job_test.py index ea9d43677..0c6c53516 100644 --- a/tests/core/job_test.py +++ b/tests/core/job_test.py @@ -528,11 +528,11 @@ def test_restore_state_queued(self): mock.patch.object(self.job_scheduler.job_runs, 'get_first_queued', return_value=queued), mock.patch.object(self.job_scheduler, 'schedule'), - mock.patch.object(self.job_scheduler, '_set_callback') - ) as (get_patch, queue_patch, sched_patch, back_patch): + mock.patch.object(job.eventloop, 'call_later') + ) as (get_patch, queue_patch, sched_patch, later_patch): self.job_scheduler.restore_state() get_patch.assert_called_once_with() - back_patch.assert_called_once_with(queued, run_queued=True) + later_patch.assert_called_once_with(0, self.job_scheduler.run_job, queued, run_queued=True) sched_patch.assert_called_once_with() def test_schedule(self): diff --git a/tron/core/job.py b/tron/core/job.py index c5fd53fce..4ec732698 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -137,7 +137,7 @@ def restore_state(self): else: queued_run = self.job_runs.get_first_queued() if queued_run: - self._set_callback(queued_run, run_queued=True) + eventloop.call_later(0, self.run_job, queued_run, run_queued=True) # Ensure we have at least 1 scheduled run self.schedule() @@ -181,11 +181,11 @@ def schedule(self): return self.create_and_schedule_runs() - def _set_callback(self, job_run, run_queued=False): + def _set_callback(self, job_run): """Set a callback for JobRun to fire at the appropriate time.""" log.info("Scheduling next Jobrun for %s", self.config.name) seconds = job_run.seconds_until_run_time() - eventloop.call_later(seconds, self.run_job, job_run, run_queued=run_queued) + eventloop.call_later(seconds, self.run_job, job_run) # TODO: new class for this method def run_job(self, job_run, run_queued=False): From 746f62094fc90afbdd02894638a80cf0297390d2 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 13:28:05 -0700 Subject: [PATCH 45/48] it's a docstring, captain --- tron/serialize/runstate/tronstore/serialize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tron/serialize/runstate/tronstore/serialize.py b/tron/serialize/runstate/tronstore/serialize.py index b7b39d260..b20c58130 100644 --- a/tron/serialize/runstate/tronstore/serialize.py +++ b/tron/serialize/runstate/tronstore/serialize.py @@ -1,8 +1,6 @@ -"""Message serialization modules for tronstore. This allows for simple writing -of stdin/out with strings that can then be put back into tuples of data -for rebuilding messages. +"""Message serialization modules for tronstore. -This is also used by the SQLAlchemy store object, an option for saving state +This is mainly used by the SQLAlchemy store object, an option for saving state with tronstore, by serializing the state data into a string that's saved in a SQL database, or by deserializing strings that are saved into state data.""" import datetime From befac571f6f1d7e78841d3784304cad65d98552a Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 15:43:20 -0700 Subject: [PATCH 46/48] improved sqlstore to not use eval/repr --- .../runstate/tronstore/store_test.py | 2 +- tron/serialize/runstate/tronstore/store.py | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/serialize/runstate/tronstore/store_test.py b/tests/serialize/runstate/tronstore/store_test.py index 398fdcd7c..1bc8ce567 100644 --- a/tests/serialize/runstate/tronstore/store_test.py +++ b/tests/serialize/runstate/tronstore/store_test.py @@ -82,7 +82,7 @@ def test_save(self): self.store.save(key, state_data, data_type) rows = self.store.engine.execute(self.store.service_table.select()) - assert_equal(rows.fetchone(), (u'dotes', unicode(repr(self.store.serializer.serialize(state_data))), u'pickle')) + assert_equal(rows.fetchone(), ('dotes', self.store.serializer.serialize(state_data), 'pickle')) def test_restore_success(self): data_type = runstate.JOB_STATE diff --git a/tron/serialize/runstate/tronstore/store.py b/tron/serialize/runstate/tronstore/store.py index 89b07bbb6..e5d478815 100644 --- a/tron/serialize/runstate/tronstore/store.py +++ b/tron/serialize/runstate/tronstore/store.py @@ -60,26 +60,28 @@ def __init__(self, name, connection_details, serializer): self.serializer = serializer self.engine = sql.create_engine(connection_details, connect_args={'check_same_thread': False}, - poolclass=sql.pool.StaticPool) + poolclass=sql.pool.StaticPool, + encoding='ascii') + self.engine.raw_connection().connection.text_factory = str self._setup_tables() def _setup_tables(self): self._metadata = sql.MetaData() self.job_state_table = sql.Table('job_state_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text), + sql.Column('state_data', sql.LargeBinary), sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.service_table = sql.Table('service_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text), + sql.Column('state_data', sql.LargeBinary), sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.job_run_table = sql.Table('job_run_data', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text), + sql.Column('state_data', sql.LargeBinary), sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self.metadata_table = sql.Table('metadata_table', self._metadata, sql.Column('key', sql.String(MAX_IDENTIFIER_LENGTH), primary_key=True), - sql.Column('state_data', sql.Text), + sql.Column('state_data', sql.LargeBinary), sql.Column('serial_method', sql.String(MAX_IDENTIFIER_LENGTH))) self._metadata.create_all(self.engine) @@ -107,7 +109,7 @@ def save(self, key, state_data, data_type): table = self._get_table(data_type) if table is None: return False - state_data = unicode(repr(self.serializer.serialize(state_data))) + state_data = self.serializer.serialize(state_data) serial_method = self.serializer.name update_result = conn.execute( table.update() @@ -132,13 +134,13 @@ def restore(self, key, data_type): ).fetchone() if not result: return (False, None) - elif str(result[1]) != self.serializer.name: + elif result[1] != self.serializer.name: # TODO: If/when we have logging in the Tronstore process, # log here that the db_store_method was different - serializer = serialize_class_map[str(result[1])] - return (True, serializer.deserialize(eval(str(result[0])))) + serializer = serialize_class_map[result[1]] + return (True, serializer.deserialize(result[0])) else: - return (True, self.serializer.deserialize(eval(str(result[0])))) + return (True, self.serializer.deserialize(result[0])) def cleanup(self): if self._connection: From 668d271344883a9df30eaadf78d6e30fdf9ba963 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Mon, 12 Aug 2013 15:57:47 -0700 Subject: [PATCH 47/48] style cleanup in jobscheduler --- tron/core/job.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tron/core/job.py b/tron/core/job.py index 4ec732698..619da3283 100644 --- a/tron/core/job.py +++ b/tron/core/job.py @@ -130,14 +130,12 @@ def __init__(self, job_runs, job_config, job_state, scheduler, actiongraph, def restore_state(self): """Restore the job state and schedule any JobRuns.""" - scheduled = [run for run in self.job_runs.get_scheduled()] + scheduled = list(self.job_runs.get_scheduled()) if scheduled: for job_run in scheduled: self._set_callback(job_run) else: - queued_run = self.job_runs.get_first_queued() - if queued_run: - eventloop.call_later(0, self.run_job, queued_run, run_queued=True) + self._run_first_queued() # Ensure we have at least 1 scheduled run self.schedule() @@ -235,6 +233,13 @@ def _queue_or_cancel_active(self, job_run): job_run.cancel() self.schedule() + def _run_first_queued(self): + # TODO: this should only start runs on the same node if this is an + # all_nodes job, but that is currently not possible + queued_run = self.job_runs.get_first_queued() + if queued_run: + eventloop.call_later(0, self.run_job, queued_run, run_queued=True) + def handle_job_events(self, _observable, event): """Handle notifications from observables. If a JobRun has completed look for queued JobRuns that may need to start now. @@ -242,11 +247,7 @@ def handle_job_events(self, _observable, event): if event != jobrun.JobRun.NOTIFY_DONE: return - # TODO: this should only start runs on the same node if this is an - # all_nodes job, but that is currently not possible - queued_run = self.job_runs.get_first_queued() - if queued_run: - eventloop.call_later(0, self.run_job, queued_run, run_queued=True) + self._run_first_queued() # Attempt to schedule a new run. This will only schedule a run if the # previous run was cancelled from a scheduled state, or if the job From a905d0bb2907cdb8f634d704d89675fefc52d300 Mon Sep 17 00:00:00 2001 From: James McGuinness Date: Tue, 13 Aug 2013 13:54:56 -0700 Subject: [PATCH 48/48] fixed inspect_serialized_state --- tools/inspect_serialized_state.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/inspect_serialized_state.py b/tools/inspect_serialized_state.py index 65d6aa951..548846cd0 100644 --- a/tools/inspect_serialized_state.py +++ b/tools/inspect_serialized_state.py @@ -34,9 +34,13 @@ def get_container(config_path): def get_state(container): config = container.get_master().state_persistence - state_manager = statemanager.PersistenceManagerFactory.from_config(config) + state_manager = statemanager.PersistentStateManager() names = container.get_job_and_service_names() - return state_manager.restore(*names) + if not state_manager.update_from_config(config): + raise SystemError('Configuration failed to load correctly.') + data = state_manager.restore(*names) + state_manager.cleanup() + return data def format_date(date_string): @@ -52,10 +56,10 @@ def max_run(item): return max(start_time) if start_time else None def build(name, job): - start_times = (max_run(job_run['runs']) for job_run in job['runs']) + start_times = (max_run(job_run['runs']) for job_run in job[1]) start_times = filter(None, start_times) last_run = format_date(max(start_times)) if start_times else None - return format % (name, job['enabled'], len(job['runs']), last_run) + return format % (name, job[0]['enabled'], len(job[1]), last_run) seq = sorted(build(*item) for item in job_states.iteritems()) return header + "".join(seq) @@ -87,4 +91,4 @@ def main(config_path, working_dir): if __name__ == "__main__": opts = parse_options() - main(opts.config_path, opts.working_dir) \ No newline at end of file + main(opts.config_path, opts.working_dir)