From 00897650297a95b481b34a4f25234cf22ebb42f4 Mon Sep 17 00:00:00 2001 From: Mike Alexander Date: Mon, 11 Jun 2018 18:12:15 -0700 Subject: [PATCH 1/4] Add page, find, and sort commands. Cleaned up the V1Query class to use Python-standard data member naming conventions for private members. Added support for using len() on the query, and a length(), both of which optimally run the base query. Standardized the sel_string and select() usage so they can both be used instead of sel_string making select() fail to function. Update the README --- README.md | 269 +++++++++++++++++++++++++++++++---------------- v1pysdk/query.py | 232 ++++++++++++++++++++++++++++++---------- 2 files changed, 359 insertions(+), 142 deletions(-) diff --git a/README.md b/README.md index 41c6647..f81f1e2 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ with V1Meta( ) as v1: user = v1.Member(20) # internal numeric ID - + print user.CreateDate, user.Name ``` @@ -41,15 +41,17 @@ with V1Meta( Asset instances are created on demand and cached so that instances with the same OID are always the same object. You can retrieve an instance by passing an asset ID to an asset class: +```python s = v1.Story(1005) - - +``` + Or by providing an OID Token: - + +```python s = v1.asset_from_oid('Story:1005') - - print s is v1.Story(1005) # True + print s is v1.Story(1005) # True +``` ### Lazyily loaded values and relations: @@ -61,15 +63,15 @@ with V1Meta( accessed for attributes that aren't currently fetched. A basic set of attributes is fetched upon the first unfound attribute. - +```python epic = v1.Epic(324355) - + # No data fetched yet. print epic #=> Epic(324355) - + # Access an attribute. print epic.Name #=> "Team Features" - + # Now some basic data has been fetched print epic #=> Epic(324355).with_data({'AssetType': 'Epic', 'Description': "Make features easier for new team members", 'AssetState': '64', @@ -77,31 +79,40 @@ with V1Meta( 'Scope_Name': 'Projects', 'Super_Name': 'New Feature Development', 'Scope': [Scope(314406)], 'SecurityScope': [Scope(314406)], 'Super': [Epic(312659)], 'Order': '-24', 'Name': 'Team Features'}) - +``` + # And further non-basic data is available, but will cause a request. + +```python print epic.CreateDate #=> '2012-05-14T23:45:14.124' - +``` + The relationship network can be traversed at will, and assets will be fetched as needed. - - # Freely traverse the relationship graph - print epic.Super.Scope.Name #=> 'Products' - + +```python + # Freely traverse the relationship graph + print epic.Super.Scope.Name #=> 'Products' +``` + Since the metadata is modeled as data, you can find the list of "Basic" attributes: - - basic_attr_names = list( v1.AttributeDefinition - .where(IsBasic = "true") - .select('Name') - .Name - ) - + +```python + basic_attr_names = list( v1.AttributeDefinition + .where(IsBasic = "true") + .select('Name') + .Name + ) +``` ### Operations: Operations on assets can be initiated by calling the appropriate method on an asset instance: - + +```python for story in epic.Subs: story.QuickSignup() - +``` + The asset instance data will be invalidated upon success, and thus re-fetched on the next attribute access. @@ -110,11 +121,12 @@ with V1Meta( The asset class is iterable to obtain all assets of that type. This is equivalent to the "query", "select" or "where" methods when given no arguments. - + +```python # WARNING: Lots of HTTP requests this way. members = list(v1.Member) # HTTP request to get the list of members. print "Members: " + ', '.join(m.Name for m in members) # HTTP request per member to fetch the Name - + # A much better way, requiring a single HTTP access via the query mechanism. members = v1.Member.select('Name') print "Members: " + ', '.join(m.Name for m in members) # HTTP request to return list of members with Name attribute. @@ -122,18 +134,28 @@ with V1Meta( # There is also a shortcut for pulling an attribute off all the results members = v1.Member.select('Name') print "Members: " + ', '.join(members.Name) - - + + # Alternative to best way with more explicit indication of what's being done + members = v1.Member.select('Name') + members.queryAll() # forces performing HTTP query to fetch all members' basic details + print "Members: " + ', '.join(m.Name for m in members) +``` + ### Queries #### Query Objects - the `select()` and `where()` methods on asset instances return a query object - upon which you can call more `.where()`'s and `.select()`'s. Iterating through + the `select()`, `where()`, and `sort()` methods on asset instances return a query object + upon which you can call more `.where()`'s, `.select()`'s, and `.sort()`'s. Iterating through the query object will run the query. - - the `.first()` method on a query object will run the query and return the first result. - + + the `.first()` or `.queryAll()` methods on a query object will run the query and return the appropriate result. + + the `find()` can be used to perform a server-side whole-word match on a field, though it's server intensive, + can only match one field, and should be used sparing. + + the `page()` can be used to limit results for the purposes of performing server-side paging. + Query results #### Simple query syntax: @@ -141,32 +163,87 @@ with V1Meta( Use `.where(Attr="value", ...)` to introduce "Equals" comparisons, and `.select("Attr", ...)` to append to the select list. - Non-"Equal" comparisons are not supported (Use the advanced query syntax). + Non-"Equal" comparisons are not supported (Use the advanced query syntax instead). +```python for s in v1.Story.where(Name='Add feature X to main product"): print s.Name, s.CreateDate, ', '.join([owner.Name for owner in s.Owners]) - + # Select only some attributes to reduce traffic - + for s in v1.Story.select('Name', 'Owners').where(Estimate='10'): print s.Name, [o.Name for o in s.Owners] - - +``` + #### Advanced query, taking the standard V1 query syntax. - The "filter" operator will take arbitrary V1 filter terms. + The `filter()` operator will take arbitrary V1 filter terms. +```python for s in (v1.Story - .filter("Estimate>'5',TotalDone.@Count<'10'") + .filter("Estimate>'5';TotalDone.@Count<'10'") .select('Name')): print s.Name +``` + +#### Limiting results from the server via paging + + It can be easier on the client to have the server perform paging by limiting the number of + results returned matching a query. Paging requires a limit on the number of items returned, and + an index of the first item in the list to return. + + The API allows the index to be left off, which assumes a default start index of 0. + +```python + pageNum = 0 + pageSize = 3 + pageStart = 0 + while True: + results = ( v1.Story + .select('Name') + .filter(str(myFilter)) + .sort('-Name') + .page(size=pageSize, start=pageStart) + ) # Requires a new query each time + if not len(results): + break; + print("Page items = " + str(len(results))) + pageNum += 1 + pageStart += pageSize + print("Page " + str(pageNum) + " : " + ', '.join(results.Name)) +``` +#### Sorting + + Sorting can be included in the query by specifying the order of the columns to sort on, and whether + those columns should be sorted ascending or descending. The default sort order is ascending. + + sort() operates like select(), where field names are listed in quotes and may be listed as separate arguments + to a single sort call, separate sort calls, or a mixture of both. + Sorting descending requires the field name to be prefaced with a dash, '-'. + Fields may only be listed in the sort order once, with repeats being ignored. + + To sort in reverse alphabetical order of names, then on Estimate time, then on Detailed Estimate time: + +```python + results = v1.Story.select('Name').filter(str(myFilter)).sort('-Name','Estimate').sort('DetailedEstimate') + print '\n'.join(results.Name) +``` +#### Matched searching + Searching, while possible, is very server intensive and should be avoided as much as possible. Server-side + searching can be whole-word matched within a single field. For this reason it should be significantly limited + with appropriate filter/where commands. +```python + results = v1.Story.select('Name').filter(str(myFilter)).find('Get a', field='Name') + print ', '.join(results.Name) #=> Get a handle on filtering, Get a toolkit for ease of use +``` #### Advanced selection, taking the standard V1 selection syntax. - The "select" operator will allow arbitrary V1 "select" terms, and will add + The `select()` operator will allow arbitrary V1 "select" terms, and will add them to the "data" mapping of the result with a key identical to the term used. - + +```python select_term = "Workitems:PrimaryWorkitem[Status='Done'].Estimate.@Sum" total_done = ( v1.Timebox .where(Name="Iteration 25") @@ -174,12 +251,13 @@ with V1Meta( ) for result in total_done: print "Total 'Done' story points: ", result.data[select_term] - +``` #### Advanced Filtering and Selection get a list of all the stories dedicated people are working on +```python writer = csv.writer(outfile) results = ( v1.Story @@ -188,12 +266,13 @@ with V1Meta( ) for result in results: writer.writerow((result['Name'], ', '.join(result['Owners.Name']))) - - +``` + ### Simple creation syntax: GOTCHA: All "required" attributes must be set, or the server will reject the data. - + +```python from v1pysdk import V1Meta v1 = V1Meta(username='admin', password='admin') new_story = v1.Story.create( @@ -204,25 +283,28 @@ with V1Meta( print new_story.CreateDate new_story.QuickSignup() print 'Owners: ' + ', '.join(o.Name for o in story.Owners) - +``` ### Simple update syntax. - Nothing is written until V1Meta.commit() is called, and then all dirty assets are written out. + Nothing is written until `V1Meta.commit()` is called, and then all dirty assets are written out. +```python story = v1.Story.where(Name='Super Cool Feature do over').first() story.Name = 'Super Cool Feature Redux' story.Owners = v1.Member.where(Name='Joe Koberg') v1.commit() # flushes all pending updates to the server +``` The V1Meta object also serves as a context manager which will commit dirty object on exit. - + +```python with V1Meta() as v1: story = v1.Story.where(Name='New Features').first() story.Owners = v1.Member.where(Name='Joe Koberg') print "Story committed implicitly." - +``` ### Attachment Contents @@ -232,18 +314,21 @@ with V1Meta( ### As Of / Historical Queries - Queries can return data "as of" a specific point in the past. The .asof() query term can + Queries can return data "as of" a specific point in the past. The `.asof()` query term can take a list (or multiple positional parameters) of timestamps or strings in ISO date format. The query is run for each timestamp in the list. A single iterable is returned that will - iterate all of the collected results. The results will all contain a data item "AsOf" with - the "As of" date of that item. Note that the "As of" date is not the date of the previous - change to the item, but rather is exactly the same date passed into the query. Also note - that timestamps such as "2012-01-01" are taken to be at the midnight starting that day, which + iterate all of the collected results. The results will all contain a data item `'AsOf'` with + the "As of" date of that item. + Note that the "As of" date is not the date of the previous change to the item, but rather is exactly the + same date passed into the query. + Also note that timestamps such as "2012-01-01" are taken to be at the midnight starting that day, which naturally excludes any activity happening during that day. You may want to specify a timestamp with a specific hour, or of the following day. - - TODO: what timezone is used? - + The timezone used when performing these comparisons is the timezone configured for the user specified + in the V1Meta object, and the time comparison is performed based on the time as determined by the + server. + +```python with V1Meta() as v1: results = (v1.Story .select("Owners") @@ -252,29 +337,30 @@ with V1Meta( ) for result in results: print result.data['AsOf'], [o.Name for o in result.Owners] - - +``` + ### Polling (TODO) A simple callback api will be available to hook asset changes - + +```python from v1meta import V1Meta from v1poll import V1Poller - + MAILBODY = """ From: VersionOne Notification To: John Smith - + Please take note of the high risk story '{0}' recently created in VersionOne. - + Link: {1} - - + + Thanks, - + Your VersionOne Software """.lstrip() - + def notify_CTO_of_high_risk_stories(story): if story.Risk > 10: import smtplib, time @@ -283,53 +369,57 @@ with V1Meta( server.quit() story.CustomNotificationLog = (story.CustomNotificationLog + "\n Notified CTO on {0}".format(time.asctime())) - + with V1Meta() as v1: with V1Poller(v1) as poller: poller.run_on_new('Story', notify_CTO_of_high_risk_stories) - + print "Notification complete and log updated." - - - + +``` + ## Performance notes An HTTP request is made to the server the first time each asset class is referenced. - + Assets do not make a request until a data item is needed from them. Further attribute access is cached if a previous request returned that attribute. Otherwise a new request is made. - + The fastest way to collect and use a set of assets is to query, with the attributes you expect to use included in the select list. The entire result set will be returned - in a single HTTP transaction - + in a single HTTP transaction if you manually call one of the methods that triggers a full query. + These methods include __iter__ (e.g. .join() uses this), __len__, and queryAll. + Writing to assets does not require reading them; setting attributes and calling the commit function does not invoke the "read" pipeline. Writing assets requires one HTTP POST per dirty asset instance. - + When an asset is committed or an operation is called, the asset data is invalidated and will - be read again on the next attribute access. + be read again on the next attribute access. Grouping your updates then calling queryAll() again + to perform a bulk update can enhance performance. ## TODO * Make things Moment-aware - + * Convert types between client and server (right now everything is a string) - + * Add debug logging - + * Beef up test coverage - + * Need to mock up server - + * Examples - + * provide an actual integration example - + * Asset creation templates and creation "in context of" other asset - + * Correctly handle multi-valued attributes including removal of values. - + + * Update to support Python 3 dynamically. + ## Installation run `python setup.py install`, or just copy the v1pysdk folder into your PYTHONPATH. @@ -337,6 +427,7 @@ run `python setup.py install`, or just copy the v1pysdk folder into your PYTHONP ## Revision History +See the GitHub history for additional follow-up history. 2013-09-27 v0.4 - A correction has been made to the multi-valued relation setter code. It used the wrong value for the XML "act" attribute, so multi-value attributes never got set correctly. Note diff --git a/v1pysdk/query.py b/v1pysdk/query.py index 9471626..ce9e2d9 100644 --- a/v1pysdk/query.py +++ b/v1pysdk/query.py @@ -8,51 +8,122 @@ class V1Query(object): def __init__(self, asset_class, sel_string=None, filterexpr=None): "Takes the asset class we will be querying" - self.asset_class = asset_class - self.where_terms = {} - self.sel_list = [] - self.asof_list = [] - self.query_results = [] - self.query_has_run = False - self.sel_string = sel_string - self.empty_sel = sel_string is None - self.where_string = filterexpr - + # warning: some of these are defined in C code + self._asset_class = asset_class + self._where_terms = {} + self._sel_list = [] + self._sel_string = None # cached copy of generated string from sel_list + self._asof_list = [] + self._query_results = [] + self._query_has_run = False + self._where_string = filterexpr + self._page_size = None + self._page_start = None + self._find_string = None + self._findIn_string = None + self._sort_list = [] + self._sort_string = None # cached copy of generated string from sort_list + self._length = 0 + + # sel_string is used when we need to query a single attribute that wasn't retrieved by default. + # it should add to any existing select list. + if sel_string: + # parse the provided sel_string using our normal select calls + self.select(sel_string.split(sep=",")) + def __iter__(self): - "Iterate over the results." - if not self.query_has_run: + """Iterate over the results, running the query the first time if necessary.""" + if not self._query_has_run: self.run_query() - for (result, asof) in self.query_results: + for (result, asof) in self._query_results: for found_asset in result.findall('Asset'): - yield self.asset_class.from_query_select(found_asset, asof) - + yield self._asset_class.from_query_select(found_asset, asof) + + def __len__(self): + """Determine the number of query results, running the query if necessary.""" + if not self._query_has_run: + self.run_query() + return self._length + + def length(self): + return len(self) + + def queryAll(self): + """Forces running of the query so the caller has the option to control when the bulk read query occurs + rather than only getting piecemeal queries as various fields are needed.""" + if not self._query_has_run: + self.run_query() + + return self + def get_sel_string(self): - if self.sel_string: - return self.sel_string - return ','.join(self.sel_list) + if not self.sel_string: + self.sel_string = ','.join(self._sel_list) + return self.sel_string + + def get_sort_string(self): + if not self._sort_string: + self._sort_string = ','.join(self._sort_list) + return self._sort_string def get_where_string(self): - terms = list("{0}='{1}'".format(attrname, criteria) for attrname, criteria in self.where_terms.items()) - if self.where_string: - terms.append(self.where_string) + terms = list("{0}='{1}'".format(attrname, criteria) for attrname, criteria in self._where_terms.items()) + if self._where_string: + terms.append(self._where_string) return ';'.join(terms) - + + def get_page_size(self): + return self._page_size + + def get_page_start(self): + return self._page_start + + def get_find_string(self): + return self._find_string + + def get_findIn_string(self): + return self._findIn_string + def run_single_query(self, url_params={}, api="Data"): urlquery = urlencode(url_params) - urlpath = '/rest-1.v1/{1}/{0}'.format(self.asset_class._v1_asset_type_name, api) + urlpath = '/rest-1.v1/{1}/{0}'.format(self._asset_class._v1_asset_type_name, api) # warning: tight coupling ahead - xml = self.asset_class._v1_v1meta.server.get_xml(urlpath, query=urlquery) + xml = self._asset_class._v1_v1meta.server.get_xml(urlpath, query=urlquery) + # xml is an elementtree::Element object so query the total of items available and determine + # the pageStart within that total set. + total = int(xml.get('total')) + pageStart = int(xml.get('pageStart')) + pageSize = int(xml.get('pageSize')) + if pageStart >= total: + # requested past end of total available + self._length = 0 + elif (total - pageStart) < pageSize: + # not enough to fill the pageSize, so length is what's left + self._length = total - pageStart + else: + # pageSize can be met, so it is + self._length = pageSize return xml - + def run_query(self): "Actually hit the server to perform the query" url_params = {} - if self.get_sel_string() or self.empty_sel: + if self.get_sel_string(): url_params['sel'] = self.get_sel_string() if self.get_where_string(): url_params['where'] = self.get_where_string() - if self.asof_list: - for asof in self.asof_list: + if self.get_sort_string(): + url_params['sort'] = self.get_sort_string() + if self.get_page_size(): + url_params['page'] = str(self.get_page_size()) + # only if page_size is set can we specify page start (optionally) + if self.get_page_start(): + url_params['page'] += "," + str(self.get_page_start()) + if self.get_find_string() and self.get_findIn_string(): + url_params['find'] = self.get_find_string() + url_params['findIn'] = self.get_findIn_string() + if self._asof_list: + for asof in self._asof_list: if asof: url_params['asof'] = str(asof) api = "Hist" @@ -60,43 +131,98 @@ def run_query(self): del url_params['asof'] api = "Data" xml = self.run_single_query(url_params, api=api) - self.query_results.append((xml, asof)) + self._query_results.append((xml, asof)) else: xml = self.run_single_query(url_params) - self.query_results.append((xml, None)) - self.query_has_run = True - + self._query_results.append((xml, None)) + self._query_has_run = True + def select(self, *args, **kw): """Add attribute names to the select list for this query. The attributes in the select list will be returned in the query results, and can be used - without further network traffic""" - - for sel in args: - parts = split_attribute(sel) - for i in range(1, len(parts)): - pname = '.'.join(parts[:i]) - if pname not in self.sel_list: - self.sel_list.append(pname) - self.sel_list.append(sel) + without further network traffic. Call with no arguments to clear select list.""" + # any calls to this invalidate our cached select string + self.sel_string=None + + if len(args) == 0: + self._sel_list = [] + else: + for sel in args: + parts = split_attribute(sel) + for i in range(1, len(parts)): + pname = '.'.join(parts[:i]) + if pname not in self._sel_list: + self._sel_list.append(pname) + self._sel_list.append(sel) return self - + + def sort(self, *args, **kw): + """Add order of fields to use for sorting. Reverse sort on that field by prefacing with a + dash (e.g. '-Name'). Call with no arguments to clear sort list.""" + # Any calls to this invalidate our cached sort string + self._sort_string=None + if len(args) == 0: + self._sort_list = [] + else: + for s in args: + labelpos=s.strip() + #if the field name is prepended with a -, strip that to determine the base field name + if labelpos[0] == '-': + labelpos=labelpos[1:] + labelneg='-' + labelpos + #only if the label in both the positive and negative sort order has never appeared before + if not (labelpos in self._sort_list) and not (labelneg in self._sort_list): + self._sort_list.append(s) + return self + def where(self, terms={}, **kw): """Add where terms to the criteria for this query. Right now this method only allows Equals comparisons.""" - self.where_terms.update(terms) - self.where_terms.update(kw) + self._where_terms.update(terms) + self._where_terms.update(kw) return self - + def filter(self, filterexpr): - self.where_string = filterexpr + self._where_string = filterexpr return self - + + def page(self, size=None, start=None): + """Add page size to limit the number returned at a time, and optionally the offset to start the page at. + 'start' is 0 based and is the index of the first record. + 'size' is a count of records to return. + Both size and start are preserved between calls, but size must be specified for either to be used in + the resulting query. + Call with no arguments to clear a previous page setting.""" + if size: + self._page_size = size + if start: + self._page_start = start + if not size and not start: + self._page_size = None + self._page_start = None + return self + + def find(self, text=None, field=None): + """A very slow and inefficient search method run on the server size to search for text fields containing + matches to the search text. + Must specify a field to search on that matches one of the defined field names or the entire search + is ignored. + Call with no arguments to clear previous find criteria.""" + if text and field: + self._find_string = str(text) + self._findIn_string = str(field) + else: + # clear old ones + self._find_string=None + self._findIn_string=None + return self + def asof(self, *asofs): - for asof_list in asofs: - if isinstance(asof_list, str): - asof_list = [asof_list] - for asof in asof_list: - self.asof_list.append(asof) + for _asof_list in asofs: + if isinstance(_asof_list, str): + _asof_list = [_asof_list] + for asof in _asof_list: + self._asof_list.append(asof) return self def first(self): @@ -117,7 +243,7 @@ def __getattr__(self, attrname): `PEP0424 `_). """ - if attrname not in self.sel_list and not attrname.startswith('__'): + if attrname not in self._sel_list and not attrname.startswith('__'): self.select(attrname) return (getattr(i, attrname) for i in self) From 47255bb759ea81847574cbd85d7140b4065d4cc7 Mon Sep 17 00:00:00 2001 From: Mike Alexander Date: Tue, 12 Jun 2018 08:25:09 -0700 Subject: [PATCH 2/4] dynamic Python3 support and updated setup.py to include Python3 checks --- setup.py | 41 ++++++++++++++-------- v1pysdk/__init__.py | 4 +-- v1pysdk/base_asset.py | 40 ++++++++++++---------- v1pysdk/cache_decorator.py | 2 +- v1pysdk/client.py | 31 ++++++++++++----- v1pysdk/query.py | 10 ++++-- v1pysdk/tests/__init__.py | 4 +-- v1pysdk/tests/connect_tests.py | 8 ++--- v1pysdk/tests/string_utils_tests.py | 53 +++++++++++++++++------------ v1pysdk/v1meta.py | 21 +++++++----- v1pysdk/v1poll.py | 2 +- v1pysdk/yamlquery.py | 32 +++++++++-------- 12 files changed, 151 insertions(+), 97 deletions(-) mode change 100644 => 100755 v1pysdk/tests/__init__.py mode change 100644 => 100755 v1pysdk/tests/connect_tests.py mode change 100644 => 100755 v1pysdk/tests/string_utils_tests.py diff --git a/setup.py b/setup.py index 7b5d540..ca5fbd5 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,41 @@ - +import sys from setuptools import setup, find_packages +installation_requirements = [ + 'elementtree', + 'testtools', + 'future', + 'urllib' +] + +if (sys.version_info < (3,0)): + # Python3 combines urllib2 and urlparse into urllib + installation_requirements.append('urllib2') + installation_requirements.append('urlparse') + # has a different name if supporting Python3 + installation_requirements.append('python-ntlm') +else: + installation_requirements.append('python-ntlm3') + setup( name = "v1pysdk", - version = "0.4", - description = "VersionOne API client", + version = "0.4", + description = "VersionOne API client", author = "Joe Koberg (VersionOne, Inc.)", author_email = "Joe.Koberg@versionone.com", license = "MIT/BSD", keywords = "versionone v1 api sdk", - url = "http://github.com/VersionOne/v1pysdk", - + url = "http://github.com/mtalexan/VersionOne.SDK.Python.git", + packages = [ - 'v1pysdk', - ], - - install_requires = [ - 'elementtree', - 'testtools', - 'iso8601', - 'python-ntlm', + 'v1pysdk-new', ], - + + install_requires = installation_requirements, + test_suite = "v1pysdk.tests", - + ) diff --git a/v1pysdk/__init__.py b/v1pysdk/__init__.py index b179654..c242700 100644 --- a/v1pysdk/__init__.py +++ b/v1pysdk/__init__.py @@ -7,6 +7,6 @@ """ -from v1meta import V1Meta -from v1poll import V1Poll +from .v1meta import V1Meta +from .v1poll import V1Poll diff --git a/v1pysdk/base_asset.py b/v1pysdk/base_asset.py index 9ab32a9..3e9b90f 100755 --- a/v1pysdk/base_asset.py +++ b/v1pysdk/base_asset.py @@ -1,12 +1,27 @@ +from future.utils import with_metaclass from pprint import pformat as pf -from query import V1Query +from .query import V1Query -class BaseAsset(object): +class IterableType(type): + "The type that's instantiated to make BaseAsset class must have an __iter__, " + "so we provide a metaclass (a thing that provides a class when instantiated) " + "that knows how to be iterated over, so we can say list(v1.Story)" + + def __iter__(Class): + for instance in Class.query(): + instance.needs_refresh = True + yield instance + +# Required dummy for with_metaclass to work properly +class DummyBaseAsset(object): + pass + +class BaseAsset(with_metaclass(IterableType,DummyBaseAsset)): """Provides common methods for the dynamically derived asset type classes built by V1Meta.asset_class""" - + @classmethod def query(Class, where=None, sel=None): 'Takes a V1 Data query string and returns an iterable of all matching items' @@ -43,22 +58,11 @@ def create(Class, **newdata): "create new asset on server and return created asset proxy instance" return Class._v1_v1meta.create_asset(Class._v1_asset_type_name, newdata) - class IterableType(type): - def __iter__(Class): - for instance in Class.query(): - instance.needs_refresh = True - yield instance - - "The type that's instantiated to make THIS class must have an __iter__, " - "so we provide a metaclass (a thing that provides a class when instantiated) " - "that knows how to be iterated over, so we can say list(v1.Story)" - __metaclass__ = IterableType - def __new__(Class, oid, moment=None): "Tries to get an instance out of the cache first, otherwise creates one" cache_key = (Class._v1_asset_type_name, oid, moment) cache = Class._v1_v1meta.global_cache - if cache.has_key(cache_key): + if cache_key in cache: self = cache[cache_key] else: self = object.__new__(Class) @@ -116,7 +120,7 @@ def repr_shallow(self, d): return pf( dict( (k, self.repr_dummy(v)) for (k,v) - in d.items() + in d.items() if v ) ) @@ -134,12 +138,12 @@ def __repr__(self): def _v1_getattr(self, attr): "Intercept access to missing attribute names. " "first return uncommitted data, then refresh if needed, then get single attr, else fail" - if self._v1_new_data.has_key(attr): + if attr in self._v1_new_data: value = self._v1_new_data[attr] else: if self._v1_needs_refresh: self._v1_refresh() - if attr not in self._v1_current_data.keys(): + if attr not in list(self._v1_current_data.keys()): self._v1_current_data[attr] = self._v1_get_single_attr(attr) value = self._v1_current_data[attr] return value diff --git a/v1pysdk/cache_decorator.py b/v1pysdk/cache_decorator.py index 11f9581..87fcac9 100755 --- a/v1pysdk/cache_decorator.py +++ b/v1pysdk/cache_decorator.py @@ -11,7 +11,7 @@ def decorator(old_f): data = {} def new_f(self, *args, **kw): new_key = keyfunc(old_f, args, kw, data) - if data.has_key(new_key): + if new_key in data: return data[new_key] new_value = old_f(self, *args, **kw) data[new_key] = new_value diff --git a/v1pysdk/client.py b/v1pysdk/client.py index f579d2f..9ef7a53 100644 --- a/v1pysdk/client.py +++ b/v1pysdk/client.py @@ -1,9 +1,21 @@ import logging, time, base64 -import urllib2 -from urllib2 import Request, urlopen, HTTPError, HTTPBasicAuthHandler, HTTPCookieProcessor -from urllib import urlencode -from urlparse import urlunparse, urlparse + +import sys +if (sys.version_info < (3,0)): + #Python2 way of doing this + import urllib2 as theUrlLib #must be a name matching the Python3 urllib.request + from urllib2 import Request, urlopen, HTTPBasicAuthHandler, HTTPCookieProcessor + from urllib import urlencode + from urlparse import urlunparse,urlparse +else: + #Python3 way of doing this + import urllib.request as theUrlLib #must be a name matching the Python2 urllib2 + import urllib.error, urllib.parse + from urllib.request import Request, urlopen, HTTPBasicAuthHandler, HTTPCookieProcessor + from urllib.error import HTTPError + from urllib.parse import urlencode + from urllib.parse import urlunparse, urlparse try: from xml.etree import ElementTree @@ -15,7 +27,10 @@ AUTH_HANDLERS = [HTTPBasicAuthHandler] try: - from ntlm.HTTPNtlmAuthHandler import HTTPNtlmAuthHandler + if (sys.version_info < (3,0)): + from ntlm.HTTPNtlmAuthHandler import HTTPNtlmAuthHandler + else: + from ntlm3.HTTPNtlmAuthHandler import HTTPNtlmAuthHandler except ImportError: logging.warn("Windows integrated authentication module (ntlm) not found.") else: @@ -70,10 +85,10 @@ def __init__(self, address="localhost", instance="VersionOne.Web", username='', def _install_opener(self): base_url = self.build_url('') - password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_manager = theUrlLib.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, base_url, self.username, self.password) handlers = [HandlerClass(password_manager) for HandlerClass in AUTH_HANDLERS] - self.opener = urllib2.build_opener(*handlers) + self.opener = theUrlLib.build_opener(*handlers) self.opener.add_handler(HTTPCookieProcessor()) def http_get(self, url): @@ -129,7 +144,7 @@ def fetch(self, path, query='', postdata=None): self._debug_headers(response.headers) self._debug_body(body, response.headers) return (None, body) - except HTTPError, e: + except HTTPError as e: if e.code == 401: raise body = e.fp.read() diff --git a/v1pysdk/query.py b/v1pysdk/query.py index ce9e2d9..c7725f3 100644 --- a/v1pysdk/query.py +++ b/v1pysdk/query.py @@ -1,5 +1,11 @@ -from urllib import urlencode -from string_utils import split_attribute +import sys + +if (sys.version_info < (3,0)): + from urllib import urlencode +else: + from urllib.parse import urlencode + +from .string_utils import split_attribute class V1Query(object): """A fluent query object. Use .select() and .where() to add items to the diff --git a/v1pysdk/tests/__init__.py b/v1pysdk/tests/__init__.py old mode 100644 new mode 100755 index e494681..9067887 --- a/v1pysdk/tests/__init__.py +++ b/v1pysdk/tests/__init__.py @@ -1,3 +1,3 @@ -import connect_tests -import meta_tests +import .connect_tests +import .meta_tests diff --git a/v1pysdk/tests/connect_tests.py b/v1pysdk/tests/connect_tests.py old mode 100644 new mode 100755 index a5eb7e9..2879d54 --- a/v1pysdk/tests/connect_tests.py +++ b/v1pysdk/tests/connect_tests.py @@ -3,13 +3,13 @@ from elementtree.ElementTree import parse, fromstring, ElementTree -from v1pysdk.client import * +from .v1pysdk.client import * class TestV1Connection(TestCase): def test_connect(self, username='admin', password='admin'): server = V1Server(address='www14.v1host.com', username=username, password=password,instance='v1sdktesting') code, body = server.fetch('/rest-1.v1/Data/Story?sel=Name') - print "\n\nCode: ", code - print "Body: ", body + print("\n\nCode: ", code) + print("Body: ", body) elem = fromstring(body) - self.assertThat(elem.tag, Equals('Assets')) \ No newline at end of file + self.assertThat(elem.tag, Equals('Assets')) diff --git a/v1pysdk/tests/string_utils_tests.py b/v1pysdk/tests/string_utils_tests.py old mode 100644 new mode 100755 index 0e8334d..7a6d4bf --- a/v1pysdk/tests/string_utils_tests.py +++ b/v1pysdk/tests/string_utils_tests.py @@ -1,33 +1,44 @@ -import unittest -from v1pysdk.string_utils import split_attribute +import unittest, sys +from .v1pysdk.string_utils import split_attribute class TestStringUtils(unittest.TestCase): + def test_split_attribute(self): - self.assertEquals(['[testing]]'],split_attribute('[testing]]')) - self.assertEquals(['[[testing]'],split_attribute('[[testing]')) - self.assertEquals(['testing','a','sentence','is','difficult'],split_attribute('testing.a.sentence.is.difficult')) - self.assertEquals(['testing','[a.sentence]','is','difficult'],split_attribute('testing.[a.sentence].is.difficult')) - self.assertEquals(['testing[.a.sentence]','is', 'difficult'],split_attribute('testing[.a.sentence].is.difficult')) - self.assertEquals(['testing','a[.sentence.]is','difficult'],split_attribute('testing.a[.sentence.]is.difficult')) - self.assertEquals(['testing','a','sentence','is','difficult]'],split_attribute('testing.a.sentence.is.difficult]')) - self.assertEquals(['testing', 'a','sentence','is',']difficult'],split_attribute('testing.a.sentence.is.]difficult')) - self.assertEquals(['[testing.a.sentence.is]','difficult'],split_attribute('[testing.a.sentence.is].difficult')) - self.assertEquals(['[testing.][a.sentence.is.difficult]'],split_attribute('[testing.][a.sentence.is.difficult]')) - self.assertEquals(['[testing]','[a]','[sentence]','[is]','[difficult]'], + self.assertEqual(['[testing]]'],split_attribute('[testing]]')) + self.assertEqual(['[[testing]'],split_attribute('[[testing]')) + self.assertEqual(['testing','a','sentence','is','difficult'],split_attribute('testing.a.sentence.is.difficult')) + self.assertEqual(['testing','[a.sentence]','is','difficult'],split_attribute('testing.[a.sentence].is.difficult')) + self.assertEqual(['testing[.a.sentence]','is', 'difficult'],split_attribute('testing[.a.sentence].is.difficult')) + self.assertEqual(['testing','a[.sentence.]is','difficult'],split_attribute('testing.a[.sentence.]is.difficult')) + self.assertEqual(['testing','a','sentence','is','difficult]'],split_attribute('testing.a.sentence.is.difficult]')) + self.assertEqual(['testing', 'a','sentence','is',']difficult'],split_attribute('testing.a.sentence.is.]difficult')) + self.assertEqual(['[testing.a.sentence.is]','difficult'],split_attribute('[testing.a.sentence.is].difficult')) + self.assertEqual(['[testing.][a.sentence.is.difficult]'],split_attribute('[testing.][a.sentence.is.difficult]')) + self.assertEqual(['[testing]','[a]','[sentence]','[is]','[difficult]'], split_attribute('[testing].[a].[sentence].[is].[difficult]')) - self.assertEquals(['testing','[[a.sentence.]is]','difficult'], + self.assertEqual(['testing','[[a.sentence.]is]','difficult'], split_attribute('testing.[[a.sentence.]is].difficult')) - self.assertEquals(["History[Status.Name='Done']"],split_attribute("History[Status.Name='Done']")) - self.assertEquals(["ParentMeAndUp[Scope.Workitems.@Count='2']"], + self.assertEqual(["History[Status.Name='Done']"],split_attribute("History[Status.Name='Done']")) + self.assertEqual(["ParentMeAndUp[Scope.Workitems.@Count='2']"], split_attribute("ParentMeAndUp[Scope.Workitems.@Count='2']") ) - self.assertEquals(["Owners","OwnedWorkitems[ChildrenMeAndDown=$]","@DistinctCount"], + self.assertEqual(["Owners","OwnedWorkitems[ChildrenMeAndDown=$]","@DistinctCount"], split_attribute("Owners.OwnedWorkitems[ChildrenMeAndDown=$].@DistinctCount") ) - self.assertEquals(["Workitems[ParentAndUp[Scope=$].@Count='1']"], + self.assertEqual(["Workitems[ParentAndUp[Scope=$].@Count='1']"], split_attribute("Workitems[ParentAndUp[Scope=$].@Count='1']") ) - self.assertEquals(["RegressionPlan","RegressionSuites[AssetState!='Dead']","TestSets[AssetState!='Dead']","Environment", "@DistinctCount"] + self.assertEqual(["RegressionPlan","RegressionSuites[AssetState!='Dead']","TestSets[AssetState!='Dead']","Environment", "@DistinctCount"] ,split_attribute("RegressionPlan.RegressionSuites[AssetState!='Dead'].TestSets[AssetState!='Dead'].Environment.@DistinctCount") ) - self.assertEquals(["Scope","ChildrenMeAndDown","Workitems:Story[ChildrenMeAndDown.ToDo.@Sum!='0.0']","Estimate","@Sum"] + self.assertEqual(["Scope","ChildrenMeAndDown","Workitems:Story[ChildrenMeAndDown.ToDo.@Sum!='0.0']","Estimate","@Sum"] ,split_attribute("Scope.ChildrenMeAndDown.Workitems:Story[ChildrenMeAndDown.ToDo.@Sum!='0.0'].Estimate.@Sum") ) +# might be needed for Python2 support +def assertEqual(arg1,arg2): + return self.assertEquals(arg1,arg2) + +# support Python2 assertEquals() method instead of assertEqual() +if (sys.version_info < (3,0)): + # define assertEqual as an unbound method on TestStringUtils so it can be mapped to assertEquals + import types + TestStringUtils.assertEqual = types.MethodType(assertEqual, None, TestStringUtils, arg1, arg2) + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/v1pysdk/v1meta.py b/v1pysdk/v1meta.py index 5a6ea07..cf9db1f 100644 --- a/v1pysdk/v1meta.py +++ b/v1pysdk/v1meta.py @@ -1,14 +1,16 @@ +import sys + try: from xml.etree import ElementTree except ImportError: from elementtree import ElementTree -from client import * -from base_asset import BaseAsset -from cache_decorator import memoized -from special_class_methods import special_classes -from none_deref import NoneDeref -from string_utils import split_attribute +from .client import * +from .base_asset import BaseAsset +from .cache_decorator import memoized +from .special_class_methods import special_classes +from .none_deref import NoneDeref +from .string_utils import split_attribute class V1Meta(object): def __init__(self, *args, **kw): @@ -88,7 +90,7 @@ def commit(self): for asset in self.dirtylist: try: asset._v1_commit() - except V1Error, e: + except V1Error as e: errors.append(e) self.dirtylist = [] return errors @@ -119,7 +121,7 @@ def generate_update_doc(self, newdata): node = Element('Attribute') node.set('name', attrname) node.set('act', 'set') - if isinstance(newvalue, unicode) != True: + if ((sys.version_info >= (3,0)) and not isinstance(newvalue, str)) or ((sys.version_info < (3,0)) and isinstance(newvalue, unicode)): node.text = str(newvalue).decode('utf-8') else: node.text = newvalue @@ -252,7 +254,8 @@ def set_attachment_blob(self, attachment, data=None): get_attachment_blob = set_attachment_blob - + + # This will eventually require iso8601 module #type_converters = dict( # Boolean = bool # Numeric = float, diff --git a/v1pysdk/v1poll.py b/v1pysdk/v1poll.py index 8210533..e0760e9 100644 --- a/v1pysdk/v1poll.py +++ b/v1pysdk/v1poll.py @@ -1,7 +1,7 @@ -from v1meta import V1Meta +from .v1meta import V1Meta import sqlite3 from collections import defaultdict diff --git a/v1pysdk/yamlquery.py b/v1pysdk/yamlquery.py index 7299b2a..826a27d 100644 --- a/v1pysdk/yamlquery.py +++ b/v1pysdk/yamlquery.py @@ -1,5 +1,9 @@ +import sys -import urllib +if (sys.version_info < (3,0)): + import urllib as theUrlLib +else: + import urllib.parse as theUrlLib import yaml def encode_v1_whereterm(input): @@ -12,11 +16,11 @@ def single_or_list(input, separator=','): return str(input) def where_terms(data): - if data.has_key("where"): + if "where" in data: for attrname, value in data['where'].items(): yield("%s='%s'"%(attrname, encode_v1_whereterm(value))) - if data.has_key("filter"): + if "filter" in data: filter = data['filter'] if isinstance(filter, list): for term in filter: @@ -29,33 +33,33 @@ def query_params(data): if wherestring: yield('where', wherestring) - if data.has_key("select"): + if "select" in data: yield('sel', single_or_list(data['select'])) - if data.has_key('asof'): + if 'asof' in data: yield('asof', data['asof']) - if data.has_key('sort'): + if 'sort' in data: yield('sort', single_or_list(data['sort'])) - if data.has_key('page'): + if 'page' in data: yield('page', "%(size)d,%(start)d"%data['page']) - if data.has_key('find'): + if 'find' in data: yield('find', data['find']) - if data.has_key('findin'): + if 'findin' in data: yield('findin', single_or_list(data['findin'])) - if data.has_key('op'): + if 'op' in data: yield('op', data['op']) def query_from_yaml(yamlstring): data = yaml.load(yamlstring) - if data and data.has_key('from'): - path = '/' + urllib.quote(data['from']) - url = path + '?' + urllib.urlencode(list(query_params(data))) + if data and 'from' in data: + path = '/' + theUrlLib.quote(data['from']) + url = path + '?' + theUrlLib.urlencode(list(query_params(data))) return url raise Exception("Invalid yaml output: " + str(data)) @@ -87,5 +91,5 @@ def query_from_yaml(yamlstring): op: Delete """ -print query_from_yaml(code) +print(query_from_yaml(code)) From 1f96ca86e7972a8e919197c5dece33433bd341c2 Mon Sep 17 00:00:00 2001 From: Mike Alexander Date: Tue, 12 Jun 2018 12:53:31 -0700 Subject: [PATCH 3/4] New release. Add reQuery and dirty tracking on query terms. max_length() to support determining total possible items when paging --- README.md | 60 ++++++++++++++++++++++++++++----- setup.py | 2 +- v1pysdk/query.py | 88 ++++++++++++++++++++++++++++++++++++------------ 3 files changed, 119 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f81f1e2..3685a3f 100644 --- a/README.md +++ b/README.md @@ -149,14 +149,17 @@ with V1Meta( upon which you can call more `.where()`'s, `.select()`'s, and `.sort()`'s. Iterating through the query object will run the query. - the `.first()` or `.queryAll()` methods on a query object will run the query and return the appropriate result. + the `.first()`, `.queryAll()`, and `.reQueryAll()` methods on a query object will run the query immediately + and return the appropriate result. the `find()` can be used to perform a server-side whole-word match on a field, though it's server intensive, can only match one field, and should be used sparing. the `page()` can be used to limit results for the purposes of performing server-side paging. - Query results + the `reQueryAll()` can be used like the `queryAll()`, but will clear all previously cached data and re-run + the HTTP query if any query options have been changed, allowing for easily repeating a query where only + response limits such as `page()` have changed. #### Simple query syntax: @@ -212,6 +215,29 @@ with V1Meta( pageStart += pageSize print("Page " + str(pageNum) + " : " + ', '.join(results.Name)) ``` + + Alternatively the `reQueryAll()` can be used to force re-querying of the content based on updated + query settings to make paging easier to implement. + +```python + pageNum = 0 + pageSize = 3 + pageStart = 0 + results = ( v1.Story + .select('Name') + .filter(str(myFilter)) + .sort('-Name') + ) + + while True: + results = results.page(size=pageSize, start=pageStart).reQueryAll() + if not len(results): + break; + pageNum += 1 + pageStart += pageSize + print("Page " + str(pageNum) + " : " + ', '.join(results.Name)) +``` + #### Sorting Sorting can be included in the query by specifying the order of the columns to sort on, and whether @@ -238,6 +264,7 @@ with V1Meta( results = v1.Story.select('Name').filter(str(myFilter)).find('Get a', field='Name') print ', '.join(results.Name) #=> Get a handle on filtering, Get a toolkit for ease of use ``` + #### Advanced selection, taking the standard V1 selection syntax. The `select()` operator will allow arbitrary V1 "select" terms, and will add @@ -385,18 +412,30 @@ with V1Meta( Assets do not make a request until a data item is needed from them. Further attribute access is cached if a previous request returned that attribute. Otherwise a new request is made. - The fastest way to collect and use a set of assets is to query, with the attributes + The fastest way to collect and use a set of assets is to query with the attributes you expect to use included in the select list. The entire result set will be returned in a single HTTP transaction if you manually call one of the methods that triggers a full query. - These methods include __iter__ (e.g. .join() uses this), __len__, and queryAll. + These methods include `__iter__()` (e.g. .join() uses this), `__len__()`, `queryAll()`, and `reQueryAll()`. Writing to assets does not require reading them; setting attributes and calling the commit function does not invoke the "read" pipeline. Writing assets requires one HTTP POST per dirty asset instance. When an asset is committed or an operation is called, the asset data is invalidated and will - be read again on the next attribute access. Grouping your updates then calling queryAll() again - to perform a bulk update can enhance performance. + be read again on the next attribute access. Grouping your updates then calling queryAll() on a fresh + query is a good way to enhance performance. + + GOTCHA: `reQueryAll()` tracks the dirty state of the query object separately from the way asset data + is invalidated following an update. Unless the terms of the query have been changed, the `reQueryAll` + won't update the cached data and a new query will be generated for each invalidated data item accessed. + To avoid this, adding and then restoring a query term on the query object can be used to cause the + re-query to actually occur. + + `reQueryAll()` can be very useful when implementing paging, changing the sorting, etc, but it should + be used with care. It clears all cached data, so any fields that were not included in the original query + and have since been retrieved are also cleared. Accessing those fields will prompt the same individual + query as before. To avoid this problem, either include the extra field(s) in your initial query, or + create a new query object for the updated query terms. ## TODO @@ -418,8 +457,6 @@ with V1Meta( * Correctly handle multi-valued attributes including removal of values. - * Update to support Python 3 dynamically. - ## Installation run `python setup.py install`, or just copy the v1pysdk folder into your PYTHONPATH. @@ -427,7 +464,12 @@ run `python setup.py install`, or just copy the v1pysdk folder into your PYTHONP ## Revision History -See the GitHub history for additional follow-up history. +2018-06-12 v0.5 - Dynamic Python3 support added. + + Add page(), sort(), queryAll(), find(), max_length(), length(), and support for len() usage to + the query objects. + + Primary repository moved to a fork that's maintained. 2013-09-27 v0.4 - A correction has been made to the multi-valued relation setter code. It used the wrong value for the XML "act" attribute, so multi-value attributes never got set correctly. Note diff --git a/setup.py b/setup.py index ca5fbd5..3337f54 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name = "v1pysdk", - version = "0.4", + version = "0.5", description = "VersionOne API client", author = "Joe Koberg (VersionOne, Inc.)", author_email = "Joe.Koberg@versionone.com", diff --git a/v1pysdk/query.py b/v1pysdk/query.py index c7725f3..5831d61 100644 --- a/v1pysdk/query.py +++ b/v1pysdk/query.py @@ -30,6 +30,8 @@ def __init__(self, asset_class, sel_string=None, filterexpr=None): self._sort_list = [] self._sort_string = None # cached copy of generated string from sort_list self._length = 0 + self._max_length = 0 # total possible number + self._dirty_query = False # sel_string is used when we need to query a single attribute that wasn't retrieved by default. # it should add to any existing select list. @@ -37,29 +39,55 @@ def __init__(self, asset_class, sel_string=None, filterexpr=None): # parse the provided sel_string using our normal select calls self.select(sel_string.split(sep=",")) - def __iter__(self): - """Iterate over the results, running the query the first time if necessary.""" + def _run_query_if_needed(self): if not self._query_has_run: self.run_query() + + def _clear_query_results(self): + """Clears the old query results so the query can be run again. Allows re-use of a query without + needing to re-create the query""" + # don't delete old results if nothing has actually changed + if self._dirty_query: + self._query_results = [] + self._query_has_run = False + + def __iter__(self): + """Iterate over the results, running the query the first time if necessary.""" + self._run_query_if_needed() for (result, asof) in self._query_results: for found_asset in result.findall('Asset'): yield self._asset_class.from_query_select(found_asset, asof) def __len__(self): """Determine the number of query results, running the query if necessary.""" - if not self._query_has_run: - self.run_query() + self._run_query_if_needed() return self._length def length(self): + """Number of query results returned. This is affected by the page() settings if they're included, + and will return the lesser of pageSize and total-pageStart. + See max_length() for a way to determine the total available responses independent of page() settings.""" return len(self) + def max_length(self): + """Returns the maximum possible number of query results, independent of page() settings. + This is the same as length() or len(self) only if pageStart=0 and pageSize=infinity.""" + self._run_query_if_needed() + return self._max_length + def queryAll(self): - """Forces running of the query so the caller has the option to control when the bulk read query occurs - rather than only getting piecemeal queries as various fields are needed.""" - if not self._query_has_run: - self.run_query() + """Forces immediate running of the query so the caller has the option to control when the bulk read + query occurs rather than only getting piecemeal queries as various fields are needed.""" + self._run_query_if_needed() + return self + def reQueryAll(self): + """Forces immediate re-running of the query so the caller has the option to control when the bulk read + query occurs rather than only getting piecemeal queries as various fields are needed. + Also allows a query object to be re-used for cases where paging is the only thing that has changed. + """ + self._clear_query_results() + self._run_query_if_needed() return self def get_sel_string(self): @@ -109,6 +137,7 @@ def run_single_query(self, url_params={}, api="Data"): else: # pageSize can be met, so it is self._length = pageSize + self._maxlength = total return xml def run_query(self): @@ -142,16 +171,19 @@ def run_query(self): xml = self.run_single_query(url_params) self._query_results.append((xml, None)) self._query_has_run = True + self._dirty_query = False # results now match the query def select(self, *args, **kw): """Add attribute names to the select list for this query. The attributes in the select list will be returned in the query results, and can be used without further network traffic. Call with no arguments to clear select list.""" + # any calls to this invalidate our cached select string self.sel_string=None - if len(args) == 0: - self._sel_list = [] + if len(self._sel_list) > 0: + self._sel_list = [] + self._dirty_query = True else: for sel in args: parts = split_attribute(sel) @@ -160,6 +192,7 @@ def select(self, *args, **kw): if pname not in self._sel_list: self._sel_list.append(pname) self._sel_list.append(sel) + self._dirty_query = True return self def sort(self, *args, **kw): @@ -168,7 +201,9 @@ def sort(self, *args, **kw): # Any calls to this invalidate our cached sort string self._sort_string=None if len(args) == 0: - self._sort_list = [] + if len(self._sort_list) > 0: + self._sort_list = [] + self._dirty_query = True else: for s in args: labelpos=s.strip() @@ -179,6 +214,7 @@ def sort(self, *args, **kw): #only if the label in both the positive and negative sort order has never appeared before if not (labelpos in self._sort_list) and not (labelneg in self._sort_list): self._sort_list.append(s) + self._dirty_query = True return self def where(self, terms={}, **kw): @@ -186,10 +222,12 @@ def where(self, terms={}, **kw): only allows Equals comparisons.""" self._where_terms.update(terms) self._where_terms.update(kw) + self._dirty_query = True return self def filter(self, filterexpr): self._where_string = filterexpr + self._dirty_query = True return self def page(self, size=None, start=None): @@ -199,13 +237,17 @@ def page(self, size=None, start=None): Both size and start are preserved between calls, but size must be specified for either to be used in the resulting query. Call with no arguments to clear a previous page setting.""" - if size: + if size and self._page_size != size: self._page_size = size - if start: + self._dirty_query = True + if start and self._page_start != start: self._page_start = start + self._dirty_query = True if not size and not start: - self._page_size = None - self._page_start = None + if self._page_size or self._page_start: + self._page_size = None + self._page_start = None + self._dirty_query = True return self def find(self, text=None, field=None): @@ -215,12 +257,15 @@ def find(self, text=None, field=None): is ignored. Call with no arguments to clear previous find criteria.""" if text and field: - self._find_string = str(text) - self._findIn_string = str(field) - else: - # clear old ones - self._find_string=None - self._findIn_string=None + if self._find_string != str(text) or self._findIn_string != str(field): + self._dirty_query = True + self._find_string = str(text) + self._findIn_string = str(field) + elif self._find_string or self._findIn_string: + self._dirty_query = True + # clear old values + self._find_string=None + self._findIn_string=None return self def asof(self, *asofs): @@ -229,6 +274,7 @@ def asof(self, *asofs): _asof_list = [_asof_list] for asof in _asof_list: self._asof_list.append(asof) + self._dirty_query = True return self def first(self): From 39e38862e523fe2c2e8096ff9c17ed320ba692bb Mon Sep 17 00:00:00 2001 From: Mike Alexander Date: Tue, 12 Jun 2018 16:46:49 -0700 Subject: [PATCH 4/4] Fix setup.py to only include minimal dependencies for python2 and python3. Remove tests since they don't work --- setup.py | 25 ++++++++++++------------- v1pysdk/tests/__init__.py | 4 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 3337f54..f844b3c 100644 --- a/setup.py +++ b/setup.py @@ -2,21 +2,15 @@ import sys from setuptools import setup, find_packages -installation_requirements = [ - 'elementtree', - 'testtools', - 'future', - 'urllib' +install_requires = [ + 'future' ] if (sys.version_info < (3,0)): - # Python3 combines urllib2 and urlparse into urllib - installation_requirements.append('urllib2') - installation_requirements.append('urlparse') # has a different name if supporting Python3 - installation_requirements.append('python-ntlm') + install_requires.append('python-ntlm') else: - installation_requirements.append('python-ntlm3') + install_requires.append('python-ntlm3') setup( name = "v1pysdk", @@ -29,12 +23,17 @@ url = "http://github.com/mtalexan/VersionOne.SDK.Python.git", packages = [ - 'v1pysdk-new', + 'v1pysdk', ], - install_requires = installation_requirements, + install_requires = install_requires, - test_suite = "v1pysdk.tests", + #tests don't work, so ignore them + #tests_require = [ + # 'testtools' + #], + # + #test_suite = "v1pysdk.tests", ) diff --git a/v1pysdk/tests/__init__.py b/v1pysdk/tests/__init__.py index 9067887..e494681 100755 --- a/v1pysdk/tests/__init__.py +++ b/v1pysdk/tests/__init__.py @@ -1,3 +1,3 @@ -import .connect_tests -import .meta_tests +import connect_tests +import meta_tests