Adding incremental refresh option for workbook and datasource endpoints along with the new JobItem code changes (#1585)

* Adding incremental refresh option to workbook and datasource along with new job item finish code

* Fix build pipeline failures related to mypy & black

Author: renoyjohnm

tableauserverclient/models/job_item.py

@@ -82,6 +82,7 @@ class FinishCode:
         Success: int = 0
         Failed: int = 1
         Cancelled: int = 2
+        Completed: int = 3
 
     def __init__(
         self,

tableauserverclient/server/endpoint/datasources_endpoint.py

@@ -340,8 +340,7 @@ def refresh(self, datasource_item: DatasourceItem, incremental: bool = False) ->
         """
         id_ = getattr(datasource_item, "id", datasource_item)
         url = f"{self.baseurl}/{id_}/refresh"
-        # refresh_req = RequestFactory.Task.refresh_req(incremental)
-        refresh_req = RequestFactory.Empty.empty_req()
+        refresh_req = RequestFactory.Task.refresh_req(incremental, self.parent_srv)
         server_response = self.post_request(url, refresh_req)
         new_job = JobItem.from_response(server_response.content, self.parent_srv.namespace)[0]
         return new_job

tableauserverclient/server/endpoint/jobs_endpoint.py

@@ -188,7 +188,7 @@ def wait_for_job(self, job_id: Union[str, JobItem], *, timeout: Optional[float]
 
         logger.info(f"Job {job_id} Completed: Finish Code: {job.finish_code} - Notes:{job.notes}")
 
-        if job.finish_code == JobItem.FinishCode.Success:
+        if job.finish_code in [JobItem.FinishCode.Success, JobItem.FinishCode.Completed]:
             return job
         elif job.finish_code == JobItem.FinishCode.Failed:
             raise JobFailedException(job)

tableauserverclient/server/endpoint/workbooks_endpoint.py

@@ -140,7 +140,7 @@ def refresh(self, workbook_item: Union[WorkbookItem, str], incremental: bool = F
         """
         id_ = getattr(workbook_item, "id", workbook_item)
         url = f"{self.baseurl}/{id_}/refresh"
-        refresh_req = RequestFactory.Task.refresh_req(incremental)
+        refresh_req = RequestFactory.Task.refresh_req(incremental, self.parent_srv)
         server_response = self.post_request(url, refresh_req)
         new_job = JobItem.from_response(server_response.content, self.parent_srv.namespace)[0]
         return new_job

tableauserverclient/server/request_factory.py

@@ -1118,11 +1118,17 @@ def run_req(self, xml_request: ET.Element, task_item: Any) -> None:
         pass
 
     @_tsrequest_wrapped
-    def refresh_req(self, xml_request: ET.Element, incremental: bool = False) -> bytes:
-        task_element = ET.SubElement(xml_request, "extractRefresh")
-        if incremental:
-            task_element.attrib["incremental"] = "true"
-        return ET.tostring(xml_request)
+    def refresh_req(
+        self, xml_request: ET.Element, incremental: bool = False, parent_srv: Optional["Server"] = None
+    ) -> Optional[bytes]:
+        if parent_srv is not None and parent_srv.check_at_least_version("3.25"):
+            task_element = ET.SubElement(xml_request, "extractRefresh")
+            if incremental:
+                task_element.attrib["incremental"] = "true"
+            return ET.tostring(xml_request)
+        elif incremental:
+            raise ValueError("Incremental refresh is only supported in 3.25+")
+        return None
 
     @_tsrequest_wrapped
     def create_extract_req(self, xml_request: ET.Element, extract_item: "TaskItem") -> bytes:

test/assets/job_get_by_id_completed.xml

@@ -0,0 +1,14 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-3.1.xsd">
+    <job id="2eef4225-aa0c-41c4-8662-a76d89ed7336" mode="job-mode" type="extractRefreshJob"
+         progress="100"     createdAt="2020-05-13T20:23:45Z"     updatedAt="2020-05-13T20:25:18Z"
+         completedAt="2020-05-13T20:25:18Z"     finishCode="3">
+        <extractRefreshJob>
+            <notes>Job detail notes</notes>
+        </extractRefreshJob>
+        <statusNotes>
+            <statusNote type="CountOfUsersAddedToGroup" value="5" text="Description of how many users were added to the group during the import." />More detail
+        </statusNotes>
+    </job>
+</tsResponse>

test/request_factory/test_task_requests.py

@@ -0,0 +1,48 @@
+import unittest
+import xml.etree.ElementTree as ET
+from unittest.mock import Mock
+from tableauserverclient.server.request_factory import TaskRequest
+
+
+class TestTaskRequest(unittest.TestCase):
+
+    def setUp(self):
+        self.task_request = TaskRequest()
+        self.xml_request = ET.Element("tsRequest")
+
+    def test_refresh_req_default(self):
+        result = self.task_request.refresh_req()
+        self.assertEqual(result, ET.tostring(self.xml_request))
+
+    def test_refresh_req_incremental(self):
+        with self.assertRaises(ValueError):
+            self.task_request.refresh_req(incremental=True)
+
+    def test_refresh_req_with_parent_srv_version_3_25(self):
+        parent_srv = Mock()
+        parent_srv.check_at_least_version.return_value = True
+        result = self.task_request.refresh_req(incremental=True, parent_srv=parent_srv)
+        expected_xml = ET.Element("tsRequest")
+        task_element = ET.SubElement(expected_xml, "extractRefresh")
+        task_element.attrib["incremental"] = "true"
+        self.assertEqual(result, ET.tostring(expected_xml))
+
+    def test_refresh_req_with_parent_srv_version_3_25_non_incremental(self):
+        parent_srv = Mock()
+        parent_srv.check_at_least_version.return_value = True
+        result = self.task_request.refresh_req(incremental=False, parent_srv=parent_srv)
+        expected_xml = ET.Element("tsRequest")
+        ET.SubElement(expected_xml, "extractRefresh")
+        self.assertEqual(result, ET.tostring(expected_xml))
+
+    def test_refresh_req_with_parent_srv_version_below_3_25(self):
+        parent_srv = Mock()
+        parent_srv.check_at_least_version.return_value = False
+        with self.assertRaises(ValueError):
+            self.task_request.refresh_req(incremental=True, parent_srv=parent_srv)
+
+    def test_refresh_req_with_parent_srv_version_below_3_25_non_incremental(self):
+        parent_srv = Mock()
+        parent_srv.check_at_least_version.return_value = False
+        result = self.task_request.refresh_req(incremental=False, parent_srv=parent_srv)
+        self.assertEqual(result, ET.tostring(self.xml_request))

test/test_job.py

@@ -11,6 +11,7 @@
 
 GET_XML = "job_get.xml"
 GET_BY_ID_XML = "job_get_by_id.xml"
+GET_BY_ID_COMPLETED_XML = "job_get_by_id_completed.xml"
 GET_BY_ID_FAILED_XML = "job_get_by_id_failed.xml"
 GET_BY_ID_CANCELLED_XML = "job_get_by_id_cancelled.xml"
 GET_BY_ID_INPROGRESS_XML = "job_get_by_id_inprogress.xml"
@@ -87,6 +88,17 @@ def test_wait_for_job_finished(self) -> None:
             self.assertEqual(job_id, job.id)
             self.assertListEqual(job.notes, ["Job detail notes"])
 
+    def test_wait_for_job_completed(self) -> None:
+        # Waiting for a bridge (cloud) job completion
+        response_xml = read_xml_asset(GET_BY_ID_COMPLETED_XML)
+        job_id = "2eef4225-aa0c-41c4-8662-a76d89ed7336"
+        with mocked_time(), requests_mock.mock() as m:
+            m.get(f"{self.baseurl}/{job_id}", text=response_xml)
+            job = self.server.jobs.wait_for_job(job_id)
+
+            self.assertEqual(job_id, job.id)
+            self.assertListEqual(job.notes, ["Job detail notes"])
+
     def test_wait_for_job_failed(self) -> None:
         # Waiting for a failed job raises an exception
         response_xml = read_xml_asset(GET_BY_ID_FAILED_XML)