include_archived as a config setting

Author: Rohit Chatterjee

airbyte-integrations/connectors/source-commcare/source_commcare/source.py

@@ -255,6 +255,7 @@ def __init__(self, start_date, app_id, name, xmlns, schema, **kwargs):
         self.streamname = name
         self.xmlns = xmlns
         self.schema = schema
+        self.include_archived = kwargs.get("include_archived", False)
 
     @property
     def name(self):
@@ -279,7 +280,7 @@ def request_params(
             "indexed_on_start": ix.strftime(self.dateformat_for_query),
             "order_by": "indexed_on",
             "limit": "1000",
-            "include_archived": True,
+            "include_archived": self.include_archived,
             "xmlns": self.xmlns,
         }
         if next_page_token:
@@ -366,6 +367,7 @@ def generate_streams(self, args, config, appdata):
             "start_date": config["start_date"],
             "form_fields_to_exclude": form_fields_to_exclude,
             "project_space": config["project_space"],
+            "include_archived": config["include_archived"],
             **args,
         }
         streams = []

airbyte-integrations/connectors/source-commcare/source_commcare/spec.yaml

@@ -42,3 +42,9 @@ connectionSpecification:
       title: Form Fields to be Excluded
       description: Forms field which you don't want to pull
       order: 4
+    include_archived:
+      type: boolean
+      title: Include Archived Forms
+      description: Include archived forms in the replication
+      default: false
+      order: 5

airbyte-integrations/connectors/source-commcare/unit_tests/test_source.py

@@ -9,8 +9,12 @@
 
 
 @pytest.fixture(name="config")
-def config_fixture():
-    return {"api_key": "apikey", "app_id": "appid", "project_space": "project_space", "start_date": "2022-01-01T00:00:00Z",'form_fields_to_exclude':[]}
+def config_fixture_1():
+    return {"api_key": "apikey", "app_id": "appid", "project_space": "project_space", "start_date": "2022-01-01T00:00:00Z",'form_fields_to_exclude':[], "include_archived": False}
+
+@pytest.fixture(name="config_include_archived")
+def config_fixture_2():
+    return {"api_key": "apikey", "app_id": "appid", "project_space": "project_space", "start_date": "2022-01-01T00:00:00Z",'form_fields_to_exclude':[], "include_archived": True}
 
 
 @patch("source_commcare.source.Application.read_records")
@@ -30,3 +34,61 @@ def test_check_connection_fail(mocker, config):
     logger_mock = MagicMock()
     excepted_outcome = " Invalid apikey, project_space or app_id : 'api_key'"
     assert source.check_connection(logger_mock, config={}) == (False, excepted_outcome)
+
+def test_include_archived_false(config):
+    source = SourceCommcare()
+    appdata = [
+        {
+            "modules": [
+                {
+                    "forms": [
+                        {
+                            "xmlns": "namespace",
+                            "name": {"en": "english_name"},
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+    streams = source.generate_streams({}, config, appdata)
+    assert len(streams) == 2
+
+    # form
+    assert streams[0].name == "english_name"
+    assert streams[0].xmlns == "namespace"
+    assert streams[0].include_archived == False
+
+    # case 
+    assert streams[1].app_id == "appid"
+    assert streams[1].project_space == "project_space"
+    assert streams[1].start_date == "2022-01-01T00:00:00Z"
+
+def test_include_archived_true(config_include_archived):
+    source = SourceCommcare()
+    appdata = [
+        {
+            "modules": [
+                {
+                    "forms": [
+                        {
+                            "xmlns": "namespace",
+                            "name": {"en": "english_name"},
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+    streams = source.generate_streams({}, config_include_archived, appdata)
+    assert len(streams) == 2
+
+    # form
+    assert streams[0].name == "english_name"
+    assert streams[0].xmlns == "namespace"
+    assert streams[0].include_archived == True
+
+    # case 
+    assert streams[1].app_id == "appid"
+    assert streams[1].project_space == "project_space"
+    assert streams[1].start_date == "2022-01-01T00:00:00Z"