From 84358b359bd41f9307948a6868da25788e5a27fd Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Tue, 1 Aug 2023 16:57:14 +0530 Subject: [PATCH 01/14] Avni Intial Commit --- .../connectors/source-avni/.dockerignore | 6 + .../connectors/source-avni/Dockerfile | 38 ++++ .../connectors/source-avni/README.md | 138 ++++++++++++++ .../source-avni/acceptance-test-config.yml | 31 +++ .../source-avni/acceptance-test-docker.sh | 3 + .../connectors/source-avni/build.gradle | 9 + .../source-avni/integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 14 ++ .../integration_tests/acceptance.py | 16 ++ .../integration_tests/configured_catalog.json | 60 ++++++ .../integration_tests/invalid_config.json | 5 + .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 14 ++ .../connectors/source-avni/main.py | 13 ++ .../connectors/source-avni/metadata.yaml | 19 ++ .../connectors/source-avni/requirements.txt | 2 + .../connectors/source-avni/setup.py | 30 +++ .../source-avni/source_avni/__init__.py | 8 + .../source_avni/schemas/encounters.json | 96 ++++++++++ .../schemas/programEncounters.json | 105 ++++++++++ .../schemas/programEnrolments.json | 92 +++++++++ .../source_avni/schemas/subjects.json | 109 +++++++++++ .../source-avni/source_avni/source.py | 180 ++++++++++++++++++ .../source-avni/source_avni/spec.yaml | 23 +++ .../source-avni/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 61 ++++++ .../source-avni/unit_tests/test_source.py | 38 ++++ .../source-avni/unit_tests/test_streams.py | 97 ++++++++++ docs/integrations/sources/avni.md | 47 +++++ 29 files changed, 1265 insertions(+) create mode 100644 airbyte-integrations/connectors/source-avni/.dockerignore create mode 100644 airbyte-integrations/connectors/source-avni/Dockerfile create mode 100644 airbyte-integrations/connectors/source-avni/README.md create mode 100644 airbyte-integrations/connectors/source-avni/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-avni/build.gradle create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-avni/main.py create mode 100644 airbyte-integrations/connectors/source-avni/metadata.yaml create mode 100644 airbyte-integrations/connectors/source-avni/requirements.txt create mode 100644 airbyte-integrations/connectors/source-avni/setup.py create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/__init__.py create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/source.py create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/spec.yaml create mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/avni.md diff --git a/airbyte-integrations/connectors/source-avni/.dockerignore b/airbyte-integrations/connectors/source-avni/.dockerignore new file mode 100644 index 0000000000000..3fcbe3fc3f0bb --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_avni +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile new file mode 100644 index 0000000000000..81bafe51cb7c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_avni ./source_avni + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-avni diff --git a/airbyte-integrations/connectors/source-avni/README.md b/airbyte-integrations/connectors/source-avni/README.md new file mode 100644 index 0000000000000..b49197bce903b --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/README.md @@ -0,0 +1,138 @@ +# Avni Source + +This is the repository for the Avni source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/avni). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-avni:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/avni) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_avni/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source avni test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-avni:dev +``` + +If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: +```bash +CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +``` + + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-avni:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-avni:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-avni:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-avni:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-avni:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-avni:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-avni:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml new file mode 100644 index 0000000000000..da3132808563f --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml @@ -0,0 +1,31 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-avni:0.1.0 +acceptance_tests: + spec: + tests: + - spec_path: "source_avni/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh new file mode 100755 index 0000000000000..b6d65deeccb43 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-avni/build.gradle b/airbyte-integrations/connectors/source-avni/build.gradle new file mode 100644 index 0000000000000..8b03324998061 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_avni' +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/__init__.py b/airbyte-integrations/connectors/source-avni/integration_tests/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..25197515a9d02 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json @@ -0,0 +1,14 @@ +{ + "subjects": { + "Last modified at":"2200-06-27T04:18:36.914Z" + }, + "programEnrolments": { + "Last modified at":"2200-06-27T04:18:36.914Z" + }, + "programEncounters": { + "Last modified at":"2200-06-27T04:18:36.914Z" + }, + "encounters": { + "Last modified at":"2200-06-27T04:18:36.914Z" + } +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py new file mode 100644 index 0000000000000..9e6409236281f --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..e7e42dcc322f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json @@ -0,0 +1,60 @@ +{ + "streams": [ + { + "stream": { + "name": "subjects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["Last modified at"], + "source_defined_primary_key": [["ID"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "cursor_field": ["audit","Last modified at"], + "primary_key": [["ID"]] + }, + { + "stream": { + "name": "programEnrolments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["Last modified at"], + "source_defined_primary_key": [["ID"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "cursor_field": ["audit","Last modified at"], + "primary_key": [["ID"]] + }, + { + "stream": { + "name": "programEncounters", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["Last modified at"], + "source_defined_primary_key": [["ID"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "cursor_field": ["audit","Last modified at"], + "primary_key": [["ID"]] + }, + { + "stream": { + "name": "encounters", + "json_schema": {}, + "supported_sync_modes": ["full_refresh","incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["Last modified at"], + "source_defined_primary_key": [["ID"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + "cursor_field": ["audit","Last modified at"], + "primary_key": [["ID"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-avni/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..6ab0009045de1 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "username": "avni", + "password": "test", + "start_date": "2000-06-27T04:18:36.914Z" +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-avni/integration_tests/sample_config.json new file mode 100644 index 0000000000000..37c2f075deaa4 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "username": "Username", + "password": "password", + "start_date": "2000-06-27T04:18:36.914Z" +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json new file mode 100644 index 0000000000000..7a8c5a6aa518d --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json @@ -0,0 +1,14 @@ +{ + "subjects": { + "Last modified at":"2000-06-27T04:18:36.914Z" + }, + "programEnrolments": { + "Last modified at":"2000-06-27T04:18:36.914Z" + }, + "programEncounters": { + "Last modified at":"2000-06-27T04:18:36.914Z" + }, + "encounters": { + "Last modified at":"2000-06-27T04:18:36.914Z" + } +} diff --git a/airbyte-integrations/connectors/source-avni/main.py b/airbyte-integrations/connectors/source-avni/main.py new file mode 100644 index 0000000000000..5ab8e86addc58 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_avni import SourceAvni + +if __name__ == "__main__": + source = SourceAvni() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml new file mode 100644 index 0000000000000..8834761d1e3c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/metadata.yaml @@ -0,0 +1,19 @@ +data: + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + oss: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: a4adf548-9f40-4eb7-958f-9ff322abd481 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-avni + githubIssueLabel: source-avni + icon: avni.svg + license: MIT + name: Avni + releaseStage: alpha + supportUrl: https://docs.airbyte.com/integrations/sources/avni +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-avni/requirements.txt b/airbyte-integrations/connectors/source-avni/requirements.txt new file mode 100644 index 0000000000000..cc57334ef619a --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-avni/setup.py b/airbyte-integrations/connectors/source-avni/setup.py new file mode 100644 index 0000000000000..9f3fd1532f27f --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", + "boto3==1.18.0", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_avni", + description="Source implementation for Avni.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/__init__.py b/airbyte-integrations/connectors/source-avni/source_avni/__init__.py new file mode 100644 index 0000000000000..93eb8dbfdf506 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAvni + +__all__ = ["SourceAvni"] diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json new file mode 100644 index 0000000000000..92fed7759f765 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": "string" + }, + "External ID": { + "type": ["null", "string"] + }, + "Voided": { + "type": "boolean" + }, + "Encounter type": { + "type": ["null", "string"] + }, + "Subject ID": { + "type": "string" + }, + "Subject type": { + "type": "string" + }, + "Subject external ID": { + "type": ["null", "string"] + }, + "Encounter date time": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Encounter location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "Earliest scheduled date": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Max scheduled date": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "observations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "Cancel location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "Cancel date time": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "cancelObservations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "audit": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Created at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": ["null", "string"] + }, + "Last modified by": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json new file mode 100644 index 0000000000000..9ed1cecd06199 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json @@ -0,0 +1,105 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": ["null", "string"] + }, + "External ID": { + "type": ["null", "string"] + }, + "Voided": { + "type": "boolean" + }, + "Subject ID": { + "type": ["null", "string"] + }, + "Subject type": { + "type": ["null", "string"] + }, + "Subject external ID": { + "type": ["null", "string"] + }, + "Enrolment ID": { + "type": ["null", "string"] + }, + "Enrolment external ID": { + "type": ["null", "string"] + }, + "Program": { + "type": ["null", "string"] + }, + "Encounter type": { + "type": ["null", "string"] + }, + "Encounter date time": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Encounter location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "Earliest scheduled date": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Max scheduled date": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "observations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "Cancel location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"], + "example": 74.7364501 + } + } + }, + "Cancel date time": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "cancelObservations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "audit": { + "type": ["null", "object"], + "properties": { + "Created at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": ["null", "string"] + }, + "Last modified by": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json new file mode 100644 index 0000000000000..9b1493442d8df --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json @@ -0,0 +1,92 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": "string" + }, + "External ID": { + "type": ["null", "string"] + }, + "Voided": { + "type": "boolean" + }, + "Subject ID": { + "type": "string" + }, + "Subject type": { + "type": "string" + }, + "Subject external ID": { + "type": ["null", "string"] + }, + "Program": { + "type": ["null", "string"] + }, + "Enrolment datetime": { + "type": ["null", "string"] + }, + "Enrolment location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "Exit datetime": { + "type": ["null", "string"] + }, + "Exit location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "observations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "exitObservations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "encounters": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "audit": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Created at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": ["null", "string"] + }, + "Last modified by": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json new file mode 100644 index 0000000000000..a6f3c23742319 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json @@ -0,0 +1,109 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": "string" + }, + "External ID": { + "type": ["null", "string"] + }, + "Voided": { + "type": "boolean" + }, + "Subject type": { + "type": ["null", "string"] + }, + "Registration location": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "X": { + "type": ["null", "number"] + }, + "Y": { + "type": ["null", "number"] + } + } + }, + "Registration date": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "object"], + "additionalProperties": true + }, + "relatives": { + "type": ["null", "array"], + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "Voided": { + "type": "boolean" + }, + "Relationship type": { + "type": ["null", "string"] + }, + "Relative ID": { + "type": ["null", "string"] + }, + "Relative external ID": { + "type": ["null", "string"] + }, + "Enter date": { + "type": ["null", "string"] + }, + "Exit date": { + "type": ["null", "string"] + } + } + } + }, + "observations": { + "type": ["null", "object"], + "additionalProperties": true + }, + "encounters": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"], + "format": "uuid" + } + }, + "enrolments": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"], + "format": "uuid" + } + }, + "audit": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Created at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": ["null", "string"] + }, + "Last modified by": { + "type": ["null", "string"] + } + } + }, + "Groups": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-avni/source_avni/source.py b/airbyte-integrations/connectors/source-avni/source_avni/source.py new file mode 100644 index 0000000000000..e42430e6f1745 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/source.py @@ -0,0 +1,180 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import boto3 +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.models import SyncMode + + +class Avni(HttpStream, ABC): + + url_base = "https://app.avniproject.org/api/" + primary_key = "ID" + cursor_value = None + current_page = 0 + last_record = None + + def __init__(self, start_date: str, path , auth_token: str, **kwargs): + super().__init__(**kwargs) + + self.start_date = start_date + self.auth_token = auth_token + self.stream=path + + +class AvniStream(Avni,IncrementalMixin): + + """ + + This implement diffrent Stream in Source Avni + + Api docs : https://avni.readme.io/docs/api-guide + Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 + """ + def path(self, **kwargs) -> str: + return self.stream + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {"lastModifiedDateTime": self.state["Last modified at"]} + if next_page_token: + params.update(next_page_token) + return params + + @property + def name(self) -> str: + return self.stream + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + + return {"auth-token": self.auth_token} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + + data = response.json()["content"] + if data: + self.last_record = data[-1] + + yield from data + + def update_state(self) -> None: + + if self.last_record: + updated_last_date = self.last_record["audit"]["Last modified at"] + if updated_last_date>self.state[self.cursor_field[1]]: + self.state = {self.cursor_field[1]: updated_last_date} + self.last_record = None + return None + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + + total_elements = int(response.json()["totalElements"]) + page_size = int(response.json()["pageSize"]) + + if total_elements == page_size: + self.current_page = self.current_page + 1 + return {"page": self.current_page} + + self.update_state() + + self.current_page = 0 + + return None + + state_checkpoint_interval = None + + @property + def cursor_field(self) -> List[str]: + return ["audit", "Last modified at"] + + @property + def state(self) -> Mapping[str, Any]: + + if self.cursor_value: + return {self.cursor_field[1]: self.cursor_value} + else: + return {self.cursor_field[1]: self.start_date} + + @state.setter + def state(self, value: Mapping[str, Any]): + self.cursor_value = value[self.cursor_field[1]] + self._state = value + + +class SourceAvni(AbstractSource): + + + def get_client_id(self): + + url_client = "https://app.avniproject.org/idp-details" + response = requests.get(url_client) + response.raise_for_status() + client = response.json() + return client["cognito"]["clientId"] + + def get_token(self, username: str, password: str, app_client_id: str) -> str: + + client = boto3.client("cognito-idp", region_name="ap-south-1") + response = client.initiate_auth( + ClientId=app_client_id, AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": username, "PASSWORD": password} + ) + return response["AuthenticationResult"]["IdToken"] + + + def check_connection(self, logger, config) -> Tuple[bool, any]: + + username = config["username"] + password = config["password"] + + try: + client_id = self.get_client_id() + except Exception as error: + return False, str(error) + ": Please connect With Avni Team" + + try: + + auth_token = self.get_token(username, password, client_id) + stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} + stream = AvniStream(path="subjects",**stream_kwargs).read_records(SyncMode.full_refresh) + return True, None + + except Exception as error: + return False, error + + def generate_streams(self, config: str) -> List[Stream]: + + streams = [] + username = config["username"] + password = config["password"] + + try: + client_id = self.get_client_id() + except Exception as error: + print(str(error) + ": Please connect With Avni Team") + raise error + + auth_token = self.get_token(username, password, client_id) + + endpoints =["subjects","programEnrolments","programEncounters","encounters"] + for endpoint in endpoints: + stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} + stream=AvniStream(path=endpoint,**stream_kwargs) + streams.append(stream) + + return streams + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + streams = self.generate_streams(config=config) + return streams \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml new file mode 100644 index 0000000000000..573ff4d87f358 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml @@ -0,0 +1,23 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Avni Spec + type: object + required: + - username + - password + - start_date + properties: + username: + type: string + description: Your avni platform Username + password: + type: string + description: Your avni platform password + airbyte_secret: true + start_date: + type: string + default: "2000-06-23T01:30:00.000Z" + description: Specify Date and time from which you want to fetch data + examples: + - "2000-10-31T01:30:00.000Z" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/__init__.py b/airbyte-integrations/connectors/source-avni/unit_tests/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..ce76621900b29 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_avni.source import AvniStream + + +@fixture +def patch_incremental_base_class(mocker): + + mocker.patch.object(AvniStream, "path", "v0/example_endpoint") + mocker.patch.object(AvniStream, "primary_key", "test_primary_key") + mocker.patch.object(AvniStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + expected_cursor_field = ["audit","Last modified at"] + assert stream.cursor_field == expected_cursor_field + + +def test_update_state(patch_incremental_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + stream.state = {"Last modified at":"OldDate"} + stream.last_record = {"audit": {"Last modified at":"NewDate"}} + expected_state = {"Last modified at":"NewDate"} + stream.update_state() + assert stream.state == expected_state + + +def test_stream_slices(patch_incremental_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + + mocker.patch.object(AvniStream, "cursor_field", "dummy_field") + stream = AvniStream(start_date="",auth_token="",path="") + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py new file mode 100644 index 0000000000000..ef052ab9094cd --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_avni.source import SourceAvni + + +def test_check_connection(mocker): + + mocker.patch('source_avni.source.SourceAvni.get_token').return_value = "Token" + mocker.patch('source_avni.source.requests.get').return_value.status_code = 200 + source = SourceAvni() + logger_mock = MagicMock() + config_mock = {"username": "test_user", "password": "test_password","start_date": "date"} + result, error = source.check_connection(logger_mock, config_mock) + assert result is True + + +def test_streams(mocker): + + mocker.patch('source_avni.source.SourceAvni.generate_streams').return_value = ["a","b","c","d"] + source = SourceAvni() + config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} + streams = source.streams(config_mock) + excepted_outcome = 4 + assert len(streams) == excepted_outcome + +def test_generate_streams(mocker): + + mocker.patch('source_avni.source.SourceAvni.get_token').return_value = "Token" + mocker.patch('source_avni.source.SourceAvni.get_client_id').return_value = "Token" + source = SourceAvni() + config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} + streams = source.generate_streams(config_mock) + assert len(streams)==4 + diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py new file mode 100644 index 0000000000000..2d709872ef9f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_avni.source import AvniStream + + +@pytest.fixture +def patch_base_class(mocker): + + mocker.patch.object(AvniStream, "path", "v0/example_endpoint") + mocker.patch.object(AvniStream, "primary_key", "test_primary_key") + mocker.patch.object(AvniStream, "__abstractmethods__", set()) + + +def test_request_params(mocker,patch_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page":5}} + stream.state = {"Last modified at":"AnyDate"} + expected_params = {"lastModifiedDateTime":"AnyDate","page":5} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + response_mock = MagicMock() + response_mock.json.return_value = { + "totalElements": 20, + "totalPages": 10, + "pageSize": 20 + } + + stream.current_page = 1 + inputs = {"response": response_mock} + expected_token = {"page": 2} + + assert stream.next_page_token(**inputs) == expected_token + assert stream.current_page == 2 + + +def test_parse_response(patch_base_class,mocker): + + stream = AvniStream(start_date="",auth_token="",path="") + response = MagicMock + response.content = b'{"content": [{"id": 1, "name": "John"}, {"id": 2, "name": "Jane"}]}' + + inputs = {"response": mocker.Mock(json=mocker.Mock(return_value={"content": [{"id": 1, "name": "Avni"}, {"id": 2, "name": "Airbyte"}]}))} + gen = stream.parse_response(**inputs) + assert next(gen) == {"id": 1, "name": "Avni"} + assert next(gen) == {"id": 2, "name": "Airbyte"} + + +def test_request_headers(patch_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + stream.auth_token = "Token" + expected_headers = {"auth-token":"Token"} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + + stream = AvniStream(start_date="",auth_token="",path="") + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + + response_mock = MagicMock() + response_mock.status_code = http_status + stream = AvniStream(start_date="",auth_token="",path="") + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + + response_mock = MagicMock() + stream = AvniStream(start_date="",auth_token="",path="") + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/avni.md b/docs/integrations/sources/avni.md new file mode 100644 index 0000000000000..526ad98eb3762 --- /dev/null +++ b/docs/integrations/sources/avni.md @@ -0,0 +1,47 @@ +# Avni + +This page contains the setup guide and reference information for the Avni source connector. + +## Prerequisites + +- Username of Avni account +- Password of Avni account + +## Setup guide + +### Step 1: Set up an Avni account + +1. Signup on [Avni](https://avniproject.org/) to create an account. +2. Create Forms for Subjects Registrations, Programs Enrolment, Program Encounter using Avni Web Console -> [Getting Started](https://avniproject.org/getting-started/) +3. Register Subjects, Enrol them in Program using Avni Android Application [Here](https://play.google.com/store/apps/details?id=com.openchsclient&hl=en&gl=US) + +### Step 2: Set up the Avni connector in Airbyte + +**For Airbyte Open Source:** + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New Source**. +3. On the source setup page, select **Avni** from the Source type dropdown and enter a name for this connector. +4. Enter the **username** and **password** of your Avni account +5. Enter the **lastModifiedDateTime**, ALl the data which have been updated since this time will be returned. The Value should be specified in "yyyy-MM-dd'T'HH:mm:ss.SSSz", e.g. "2000-10-31T01:30:00.000Z". If all the data needed to be fetch keep this parameter to any old date or use e.g. date. +6. Click **Set up source**. + +## Supported sync modes + +The Avni source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +​ + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- (Recommended)[ Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) + + +## Supported Streams + +Avni Source connector Support Following Streams: + +- **Subjects Stream** : This stream provides details of registered subjects. You can retrieve information about subjects who have been registered in the system. +- **Program Enrolment Stream** : This stream provides program enrolment data. You can obtain information about subjects who have enrolled in programs. +- **Program Encounter Stream**, This stream provides data about encounters that occur within programs. You can retrieve information about all the encounters that have taken place within programs. +- **Subject Encounter Stream**, This stream provides data about encounters involving subjects, excluding program encounters. You can obtain information about all the encounters that subjects have had outside of program-encounter. From 1e03cc3108d3894250465d159167da1d8c765315 Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Fri, 4 Aug 2023 09:59:06 +0530 Subject: [PATCH 02/14] Change to class implementation --- .../integration_tests/abnormal_state.json | 12 +- .../integration_tests/configured_catalog.json | 76 +++++++-- .../integration_tests/sample_state.json | 12 +- .../connectors/source-avni/metadata.yaml | 2 +- .../source_avni/schemas/encounters.json | 9 +- ...ncounters.json => program_encounters.json} | 9 +- ...nrolments.json => program_enrolments.json} | 6 +- .../source_avni/schemas/subjects.json | 10 +- .../source-avni/source_avni/source.py | 153 +++++++++++------- .../source-avni/source_avni/spec.yaml | 2 +- .../unit_tests/test_incremental_streams.py | 30 ++-- .../source-avni/unit_tests/test_source.py | 38 ++--- .../source-avni/unit_tests/test_streams.py | 16 +- docs/integrations/sources/avni.md | 5 + 14 files changed, 249 insertions(+), 131 deletions(-) rename airbyte-integrations/connectors/source-avni/source_avni/schemas/{programEncounters.json => program_encounters.json} (93%) rename airbyte-integrations/connectors/source-avni/source_avni/schemas/{programEnrolments.json => program_enrolments.json} (95%) diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json index 25197515a9d02..31f5193c1612c 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json @@ -1,14 +1,14 @@ { "subjects": { - "Last modified at":"2200-06-27T04:18:36.914Z" + "last_modified_at":"2200-06-27T04:18:36.914Z" }, - "programEnrolments": { - "Last modified at":"2200-06-27T04:18:36.914Z" + "program_enrolments": { + "last_modified_at":"2200-06-27T04:18:36.914Z" }, - "programEncounters": { - "Last modified at":"2200-06-27T04:18:36.914Z" + "program_encounters": { + "last_modified_at":"2200-06-27T04:18:36.914Z" }, "encounters": { - "Last modified at":"2200-06-27T04:18:36.914Z" + "last_modified_at":"2200-06-27T04:18:36.914Z" } } diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json index e7e42dcc322f7..df39258c9271b 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json @@ -3,57 +3,105 @@ { "stream": { "name": "subjects", - "json_schema": {}, + "json_schema": { + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + }, + "lastModifiedDateTime":{ + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, - "default_cursor_field": ["Last modified at"], + "default_cursor_field": ["last_modified_at"], "source_defined_primary_key": [["ID"]] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["audit","Last modified at"], + "cursor_field": ["last_modified_at"], "primary_key": [["ID"]] }, { "stream": { - "name": "programEnrolments", - "json_schema": {}, + "name": "program_enrolments", + "json_schema": { + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + }, + "lastModifiedDateTime":{ + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, - "default_cursor_field": ["Last modified at"], + "default_cursor_field": ["last_modified_at"], "source_defined_primary_key": [["ID"]] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["audit","Last modified at"], + "cursor_field": ["last_modified_at"], "primary_key": [["ID"]] }, { "stream": { - "name": "programEncounters", - "json_schema": {}, + "name": "program_encounters", + "json_schema": { + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + }, + "lastModifiedDateTime":{ + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, - "default_cursor_field": ["Last modified at"], + "default_cursor_field": ["last_modified_at"], "source_defined_primary_key": [["ID"]] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["audit","Last modified at"], + "cursor_field": ["last_modified_at"], "primary_key": [["ID"]] }, { "stream": { "name": "encounters", - "json_schema": {}, + "json_schema": { + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + }, + "lastModifiedDateTime":{ + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, - "default_cursor_field": ["Last modified at"], + "default_cursor_field": ["last_modified_at"], "source_defined_primary_key": [["ID"]] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["audit","Last modified at"], + "cursor_field": ["last_modified_at"], "primary_key": [["ID"]] } ] diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json index 7a8c5a6aa518d..2a47e04cc4db1 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json @@ -1,14 +1,14 @@ { "subjects": { - "Last modified at":"2000-06-27T04:18:36.914Z" + "last_modified_at":"2000-06-27T04:18:36.914Z" }, - "programEnrolments": { - "Last modified at":"2000-06-27T04:18:36.914Z" + "program_enrolments": { + "last_modified_at":"2000-06-27T04:18:36.914Z" }, - "programEncounters": { - "Last modified at":"2000-06-27T04:18:36.914Z" + "program_encounters": { + "last_modified_at":"2000-06-27T04:18:36.914Z" }, "encounters": { - "Last modified at":"2000-06-27T04:18:36.914Z" + "last_modified_at":"2000-06-27T04:18:36.914Z" } } diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml index 8834761d1e3c6..dbd0ed9f12fa4 100644 --- a/airbyte-integrations/connectors/source-avni/metadata.yaml +++ b/airbyte-integrations/connectors/source-avni/metadata.yaml @@ -4,7 +4,7 @@ data: - "*" # Please change to the hostname of the source. registries: oss: - enabled: false + enabled: true connectorSubtype: api connectorType: source definitionId: a4adf548-9f40-4eb7-958f-9ff322abd481 diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json index 92fed7759f765..c1727a9fc73b3 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json @@ -60,7 +60,8 @@ "type": ["null", "number"] }, "Y": { - "type": ["null", "number"] + "type": ["null", "number"], + "example": 74.7364501 } } }, @@ -72,6 +73,10 @@ "type": ["null", "object"], "additionalProperties": true }, + "last_modified_at":{ + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, "audit": { "type": ["null", "object"], "additionalProperties": true, @@ -93,4 +98,4 @@ } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json similarity index 93% rename from airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json rename to airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json index 9ed1cecd06199..3e1a4a25acd95 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEncounters.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json @@ -45,7 +45,8 @@ "type": ["null", "number"] }, "Y": { - "type": ["null", "number"] + "type": ["null", "number"], + "example": 74.7364501 } } }, @@ -82,6 +83,10 @@ "type": ["null", "object"], "additionalProperties": true }, + "last_modified_at":{ + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, "audit": { "type": ["null", "object"], "properties": { @@ -102,4 +107,4 @@ } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json similarity index 95% rename from airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json rename to airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json index 9b1493442d8df..9d858d0b077d4 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/programEnrolments.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json @@ -68,6 +68,10 @@ "type": ["null", "string"] } }, + "last_modified_at":{ + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, "audit": { "type": ["null", "object"], "additionalProperties": true, @@ -89,4 +93,4 @@ } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json index a6f3c23742319..8761f2514761f 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json @@ -23,7 +23,8 @@ "type": ["null", "number"] }, "Y": { - "type": ["null", "number"] + "type": ["null", "number"], + "example": 74.7364501 } } }, @@ -79,6 +80,11 @@ "format": "uuid" } }, + "last_modified_at":{ + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + } + , "audit": { "type": ["null", "object"], "additionalProperties": true, @@ -106,4 +112,4 @@ } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/source.py b/airbyte-integrations/connectors/source-avni/source_avni/source.py index e42430e6f1745..e12d819d2c8c4 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/source.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/source.py @@ -7,53 +7,36 @@ import boto3 import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.models import SyncMode -class Avni(HttpStream, ABC): +class AvniStream(HttpStream, ABC): url_base = "https://app.avniproject.org/api/" primary_key = "ID" cursor_value = None current_page = 0 last_record = None - - def __init__(self, start_date: str, path , auth_token: str, **kwargs): + + def __init__(self, start_date: str, auth_token: str, **kwargs): super().__init__(**kwargs) self.start_date = start_date self.auth_token = auth_token - self.stream=path - -class AvniStream(Avni,IncrementalMixin): - - """ - - This implement diffrent Stream in Source Avni - - Api docs : https://avni.readme.io/docs/api-guide - Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 - """ - def path(self, **kwargs) -> str: - return self.stream - def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"lastModifiedDateTime": self.state["Last modified at"]} + params = {"lastModifiedDateTime": self.state["last_modified_at"]} if next_page_token: params.update(next_page_token) return params - - @property - def name(self) -> str: - return self.stream - + def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: @@ -68,13 +51,28 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield from data + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[StreamData]: + + records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + for record in records: + last_modified_at = record["audit"]["Last modified at"] + record["last_modified_at"] = last_modified_at + yield record + def update_state(self) -> None: if self.last_record: updated_last_date = self.last_record["audit"]["Last modified at"] - if updated_last_date>self.state[self.cursor_field[1]]: - self.state = {self.cursor_field[1]: updated_last_date} + if updated_last_date > self.state["last_modified_at"]: + self.state = {self.cursor_field: updated_last_date} self.last_record = None + return None def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -92,29 +90,78 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return None + +class IncrementalAvniStream(AvniStream, IncrementalMixin, ABC): + state_checkpoint_interval = None @property - def cursor_field(self) -> List[str]: - return ["audit", "Last modified at"] + def cursor_field(self) -> str: + return "last_modified_at" @property def state(self) -> Mapping[str, Any]: if self.cursor_value: - return {self.cursor_field[1]: self.cursor_value} + return {self.cursor_field: self.cursor_value} else: - return {self.cursor_field[1]: self.start_date} + return {self.cursor_field: self.start_date} @state.setter def state(self, value: Mapping[str, Any]): - self.cursor_value = value[self.cursor_field[1]] + self.cursor_value = value[self.cursor_field] self._state = value -class SourceAvni(AbstractSource): - +class Subjects(IncrementalAvniStream): + + """ + This implement Subject Stream in Source Avni + Api docs : https://avni.readme.io/docs/api-guide + Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 + """ + + def path(self, **kwargs) -> str: + return "subjects" + + +class ProgramEnrolments(IncrementalAvniStream): + + """ + This implement ProgramEnrolments Stream in Source Avni + Api docs : https://avni.readme.io/docs/api-guide + Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 + """ + def path(self, **kwargs) -> str: + return "programEnrolments" + + +class ProgramEncounters(IncrementalAvniStream): + + """ + This implement ProgramEncounters Stream in Source Avni + Api docs : https://avni.readme.io/docs/api-guide + Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 + """ + + def path(self, **kwargs) -> str: + return "programEncounters" + + +class Encounters(IncrementalAvniStream): + + """ + This implement Encounters Stream in Source Avni + Api docs : https://avni.readme.io/docs/api-guide + Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 + """ + + def path(self, **kwargs) -> str: + return "encounters" + + +class SourceAvni(AbstractSource): def get_client_id(self): url_client = "https://app.avniproject.org/idp-details" @@ -125,15 +172,19 @@ def get_client_id(self): def get_token(self, username: str, password: str, app_client_id: str) -> str: + """ + Avni Api Authentication : https://avni.readme.io/docs/api-guide#authentication + AWS Cognito for authentication : https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cognito-idp/client/initiate_auth.html + """ + client = boto3.client("cognito-idp", region_name="ap-south-1") response = client.initiate_auth( ClientId=app_client_id, AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": username, "PASSWORD": password} ) return response["AuthenticationResult"]["IdToken"] - - + def check_connection(self, logger, config) -> Tuple[bool, any]: - + username = config["username"] password = config["password"] @@ -143,18 +194,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, str(error) + ": Please connect With Avni Team" try: - auth_token = self.get_token(username, password, client_id) stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} - stream = AvniStream(path="subjects",**stream_kwargs).read_records(SyncMode.full_refresh) + next(Subjects(**stream_kwargs).read_records(SyncMode.full_refresh)) return True, None - + except Exception as error: return False, error - - def generate_streams(self, config: str) -> List[Stream]: - - streams = [] + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + username = config["username"] password = config["password"] @@ -165,16 +214,12 @@ def generate_streams(self, config: str) -> List[Stream]: raise error auth_token = self.get_token(username, password, client_id) - - endpoints =["subjects","programEnrolments","programEncounters","encounters"] - for endpoint in endpoints: - stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} - stream=AvniStream(path=endpoint,**stream_kwargs) - streams.append(stream) - return streams - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: + stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} - streams = self.generate_streams(config=config) - return streams \ No newline at end of file + return [ + Subjects(**stream_kwargs), + ProgramEnrolments(**stream_kwargs), + ProgramEncounters(**stream_kwargs), + Encounters(**stream_kwargs), + ] diff --git a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml index 573ff4d87f358..b010d2cf15844 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.com/integrations/sources/avni connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Avni Spec diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py index ce76621900b29..ccd09b924011b 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py @@ -5,37 +5,37 @@ from airbyte_cdk.models import SyncMode from pytest import fixture -from source_avni.source import AvniStream +from source_avni.source import IncrementalAvniStream @fixture def patch_incremental_base_class(mocker): - mocker.patch.object(AvniStream, "path", "v0/example_endpoint") - mocker.patch.object(AvniStream, "primary_key", "test_primary_key") - mocker.patch.object(AvniStream, "__abstractmethods__", set()) + mocker.patch.object(IncrementalAvniStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalAvniStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalAvniStream, "__abstractmethods__", set()) def test_cursor_field(patch_incremental_base_class): - stream = AvniStream(start_date="",auth_token="",path="") - expected_cursor_field = ["audit","Last modified at"] + stream = IncrementalAvniStream(start_date="",auth_token="") + expected_cursor_field = "last_modified_at" assert stream.cursor_field == expected_cursor_field def test_update_state(patch_incremental_base_class): - stream = AvniStream(start_date="",auth_token="",path="") - stream.state = {"Last modified at":"OldDate"} - stream.last_record = {"audit": {"Last modified at":"NewDate"}} - expected_state = {"Last modified at":"NewDate"} + stream = IncrementalAvniStream(start_date="",auth_token="") + stream.state = {"last_modified_at":"2000-06-27T04:18:36.914Z"} + stream.last_record = {"audit":{"Last modified at":"2001-06-27T04:18:36.914Z"}} + expected_state = {"last_modified_at":"2001-06-27T04:18:36.914Z"} stream.update_state() assert stream.state == expected_state def test_stream_slices(patch_incremental_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = IncrementalAvniStream(start_date="",auth_token="") inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} expected_stream_slice = [None] assert stream.stream_slices(**inputs) == expected_stream_slice @@ -43,19 +43,19 @@ def test_stream_slices(patch_incremental_base_class): def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(AvniStream, "cursor_field", "dummy_field") - stream = AvniStream(start_date="",auth_token="",path="") + mocker.patch.object(IncrementalAvniStream, "cursor_field", "dummy_field") + stream = IncrementalAvniStream(start_date="",auth_token="") assert stream.supports_incremental def test_source_defined_cursor(patch_incremental_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = IncrementalAvniStream(start_date="",auth_token="") assert stream.source_defined_cursor def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = IncrementalAvniStream(start_date="",auth_token="") expected_checkpoint_interval = None assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py index ef052ab9094cd..7022f4de4ce4d 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py @@ -2,37 +2,37 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import patch from source_avni.source import SourceAvni -def test_check_connection(mocker): - - mocker.patch('source_avni.source.SourceAvni.get_token').return_value = "Token" - mocker.patch('source_avni.source.requests.get').return_value.status_code = 200 - source = SourceAvni() - logger_mock = MagicMock() - config_mock = {"username": "test_user", "password": "test_password","start_date": "date"} - result, error = source.check_connection(logger_mock, config_mock) - assert result is True +def test_check_connection_success(mocker): + with patch('source_avni.source.SourceAvni.get_client_id') as get_client_id_mock, \ + patch('source_avni.source.SourceAvni.get_token') as get_token_mock, \ + patch('source_avni.source.Subjects.read_records') as read_records_mock: + get_client_id_mock.return_value = "ClientID" + get_token_mock.return_value = "Token" + read_records_mock.return_value = iter(["record1", "record2"]) + source = SourceAvni() + config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} + result,msg = source.check_connection(None, config_mock) + assert result is True def test_streams(mocker): - mocker.patch('source_avni.source.SourceAvni.generate_streams').return_value = ["a","b","c","d"] + mocker.patch('source_avni.source.SourceAvni.get_token').return_value = 'fake_token' source = SourceAvni() config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} streams = source.streams(config_mock) excepted_outcome = 4 assert len(streams) == excepted_outcome -def test_generate_streams(mocker): - - mocker.patch('source_avni.source.SourceAvni.get_token').return_value = "Token" - mocker.patch('source_avni.source.SourceAvni.get_client_id').return_value = "Token" + +def test_get_client_id(mocker): + source = SourceAvni() - config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} - streams = source.generate_streams(config_mock) - assert len(streams)==4 - + client_id = source.get_client_id() + expected_length = 26 + assert len(client_id) == expected_length diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py index 2d709872ef9f9..c0be8d8359d45 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py @@ -19,16 +19,16 @@ def patch_base_class(mocker): def test_request_params(mocker,patch_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page":5}} - stream.state = {"Last modified at":"AnyDate"} + stream.state = {"last_modified_at":"AnyDate"} expected_params = {"lastModifiedDateTime":"AnyDate","page":5} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") response_mock = MagicMock() response_mock.json.return_value = { "totalElements": 20, @@ -46,7 +46,7 @@ def test_next_page_token(patch_base_class): def test_parse_response(patch_base_class,mocker): - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") response = MagicMock response.content = b'{"content": [{"id": 1, "name": "John"}, {"id": 2, "name": "Jane"}]}' @@ -58,7 +58,7 @@ def test_parse_response(patch_base_class,mocker): def test_request_headers(patch_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} stream.auth_token = "Token" expected_headers = {"auth-token":"Token"} @@ -67,7 +67,7 @@ def test_request_headers(patch_base_class): def test_http_method(patch_base_class): - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") expected_method = "GET" assert stream.http_method == expected_method @@ -85,13 +85,13 @@ def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = AvniStream(start_date="",auth_token="",path="") + stream = AvniStream(start_date="",auth_token="") expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/avni.md b/docs/integrations/sources/avni.md index 526ad98eb3762..8ff39909c6bf4 100644 --- a/docs/integrations/sources/avni.md +++ b/docs/integrations/sources/avni.md @@ -45,3 +45,8 @@ Avni Source connector Support Following Streams: - **Program Enrolment Stream** : This stream provides program enrolment data. You can obtain information about subjects who have enrolled in programs. - **Program Encounter Stream**, This stream provides data about encounters that occur within programs. You can retrieve information about all the encounters that have taken place within programs. - **Subject Encounter Stream**, This stream provides data about encounters involving subjects, excluding program encounters. You can obtain information about all the encounters that subjects have had outside of program-encounter. + +## Changelog + +| Version | Date | Pull Request | Subject | +| 0.1.0 | 2023-07-27 | [28141](https://github.com/airbytehq/airbyte/pull/28141) | Avni Source Connector | From 793f70053f7936e7a7914d9b588cb0185cc8f221 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Thu, 10 Aug 2023 21:09:57 +0530 Subject: [PATCH 03/14] Added title to form properties --- .../connectors/source-avni/source_avni/spec.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml index b010d2cf15844..2ecbcd34dae22 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml @@ -10,14 +10,17 @@ connectionSpecification: properties: username: type: string - description: Your avni platform Username + description: Your Avni platform username + title: Username password: type: string - description: Your avni platform password + description: Your Avni platform password + title: Password airbyte_secret: true start_date: type: string default: "2000-06-23T01:30:00.000Z" description: Specify Date and time from which you want to fetch data + title: Start date and time examples: - "2000-10-31T01:30:00.000Z" \ No newline at end of file From 8e68f67a41d21a292b766e7d1277a8bdd46f8877 Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Thu, 7 Sep 2023 14:21:00 +0530 Subject: [PATCH 04/14] migrated to lowcode --- .../connectors/source-avni/Dockerfile | 2 +- .../connectors/source-avni/README.md | 64 +---- .../source-avni/{unit_tests => }/__init__.py | 0 .../source-avni/acceptance-test-config.yml | 46 ++-- .../integration_tests/configured_catalog.json | 48 +--- .../connectors/source-avni/metadata.yaml | 14 +- .../connectors/source-avni/setup.py | 3 +- .../source-avni/source_avni/components.py | 36 +++ .../source-avni/source_avni/manifest.yaml | 141 +++++++++++ .../source_avni/schemas/subjects.json | 9 +- .../source-avni/source_avni/source.py | 227 +----------------- .../source-avni/source_avni/spec.yaml | 26 -- .../source-avni/unit_tests/test_components.py | 44 ++++ .../unit_tests/test_incremental_streams.py | 61 ----- .../source-avni/unit_tests/test_source.py | 38 --- .../source-avni/unit_tests/test_streams.py | 97 -------- 16 files changed, 282 insertions(+), 574 deletions(-) rename airbyte-integrations/connectors/source-avni/{unit_tests => }/__init__.py (100%) create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/components.py create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-avni/source_avni/spec.yaml create mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_components.py delete mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py delete mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile index 81bafe51cb7c3..09b5073bfc31c 100644 --- a/airbyte-integrations/connectors/source-avni/Dockerfile +++ b/airbyte-integrations/connectors/source-avni/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder diff --git a/airbyte-integrations/connectors/source-avni/README.md b/airbyte-integrations/connectors/source-avni/README.md index b49197bce903b..075de8d477962 100644 --- a/airbyte-integrations/connectors/source-avni/README.md +++ b/airbyte-integrations/connectors/source-avni/README.md @@ -1,35 +1,10 @@ # Avni Source -This is the repository for the Avni source connector, written in Python. +This is the repository for the Avni configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/avni). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -47,14 +22,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source avni test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -63,12 +30,6 @@ First, make sure you build the latest Docker image: docker build . -t airbyte/source-avni:dev ``` -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: -```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh -``` - - You can also build the connector image via Gradle: ``` ./gradlew :airbyte-integrations:connectors:source-avni:airbyteDocker @@ -85,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-avni:dev discover --co docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-avni:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/__init__.py b/airbyte-integrations/connectors/source-avni/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-avni/unit_tests/__init__.py rename to airbyte-integrations/connectors/source-avni/__init__.py diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml index da3132808563f..e21280f168e43 100644 --- a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml @@ -1,31 +1,31 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests -connector_image: airbyte/source-avni:0.1.0 +connector_image: airbyte/source-avni:dev acceptance_tests: - spec: - tests: - - spec_path: "source_avni/spec.yaml" - connection: - tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - tests: - - config_path: "secrets/config.json" + # spec: + # tests: + # - spec_path: "source_avni/spec.yaml" + # connection: + # tests: + # - config_path: "secrets/config.json" + # status: "succeed" + # - config_path: "integration_tests/invalid_config.json" + # status: "failed" + # discovery: + # tests: + # - config_path: "secrets/config.json" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + # full_refresh: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json index df39258c9271b..f3f7d38254c03 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json @@ -4,17 +4,7 @@ "stream": { "name": "subjects", "json_schema": { - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "lastModifiedDateTime":{ - "type": "string" - } - } + "properties": {} }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, @@ -30,17 +20,7 @@ "stream": { "name": "program_enrolments", "json_schema": { - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "lastModifiedDateTime":{ - "type": "string" - } - } + "properties": {} }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, @@ -56,17 +36,7 @@ "stream": { "name": "program_encounters", "json_schema": { - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "lastModifiedDateTime":{ - "type": "string" - } - } + "properties": {} }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, @@ -82,17 +52,7 @@ "stream": { "name": "encounters", "json_schema": { - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "lastModifiedDateTime":{ - "type": "string" - } - } + "properties": {} }, "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml index dbd0ed9f12fa4..79cdb44b311a7 100644 --- a/airbyte-integrations/connectors/source-avni/metadata.yaml +++ b/airbyte-integrations/connectors/source-avni/metadata.yaml @@ -1,19 +1,25 @@ data: allowedHosts: hosts: - - "*" # Please change to the hostname of the source. + - "*" registries: oss: enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source - definitionId: a4adf548-9f40-4eb7-958f-9ff322abd481 + definitionId: 5d297ac7-355e-4a04-be75-a5e7e175fc4e dockerImageTag: 0.1.0 dockerRepository: airbyte/source-avni githubIssueLabel: source-avni icon: avni.svg license: MIT - name: Avni + name: Avni + releaseDate: "2023-09-07" releaseStage: alpha - supportUrl: https://docs.airbyte.com/integrations/sources/avni + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/avni + tags: + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-avni/setup.py b/airbyte-integrations/connectors/source-avni/setup.py index 9f3fd1532f27f..b5f12bb51f556 100644 --- a/airbyte-integrations/connectors/source-avni/setup.py +++ b/airbyte-integrations/connectors/source-avni/setup.py @@ -6,11 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.1", "boto3==1.18.0", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py new file mode 100644 index 0000000000000..ba6d6eb99a6b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/components.py @@ -0,0 +1,36 @@ +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator +from dataclasses import dataclass +import boto3 +import requests + + +@dataclass +class CustomAuthenticator(BasicHttpAuthenticator): + + + @property + def token(self) -> str: + + username = self._username.eval(self.config) + password = self._password.eval(self.config) + + app_client_id = self.get_client_id() + + client = boto3.client("cognito-idp", region_name="ap-south-1") + response = client.initiate_auth( + ClientId=app_client_id, AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": username, "PASSWORD": password} + ) + token = response["AuthenticationResult"]["IdToken"] + return token + + @property + def auth_header(self) -> str: + return "auth-token" + + def get_client_id(self): + + url_client = "https://app.avniproject.org/idp-details" + response = requests.get(url_client) + response.raise_for_status() + client = response.json() + return client["cognito"]["clientId"] diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml new file mode 100644 index 0000000000000..d62154bead30d --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -0,0 +1,141 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["content"] + + requester: + type: HttpRequester + url_base: "https://app.avniproject.org/api" + http_method: "GET" + authenticator: + class_name: source_avni.components.CustomAuthenticator + username: "{{config['username']}}" + password: "{{config['password']}}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "size" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "last_modified_at" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_time_option: + field_name: "lastModifiedDateTime" + inject_into: "request_parameter" + + transformations_base: + - type: AddFields + fields: + - path: [ "last_modified_at" ] + value: "{{ record['audit']['Last modified at'] }}" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + subjects_stream: + $ref: "#/definitions/base_stream" + name: "subjects" + primary_key: "ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/subjects" + + program_encounters_stream: + $ref: "#/definitions/base_stream" + name: "program_encounters" + primary_key: "ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/programEncounters" + + program_enrolments_stream: + $ref: "#/definitions/base_stream" + name: "program_enrolments" + primary_key: "ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/programEnrolments" + + encounters_stream: + $ref: "#/definitions/base_stream" + name: "encounters" + primary_key: "ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/encounters" + +streams: + - "#/definitions/subjects_stream" + - "#/definitions/program_enrolments_stream" + - "#/definitions/program_encounters_stream" + - "#/definitions/encounters_stream" + +check: + type: CheckStream + stream_names: + - "subjects" + + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/avni + connection_specification: + title: Avni Spec + type: object + required: + - username + - password + - start_date + additionalProperties: true + properties: + username: + type: string + description: Your avni platform Username + password: + type: string + description: Your avni platform password + airbyte_secret: true + start_date: + type: string + default: "2000-06-23T01:30:00.000Z" + description: Specify Date and time from which you want to fetch data + examples: + - "2000-10-31T01:30:00.000Z" diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json index 8761f2514761f..30006a7dcf7b3 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json @@ -83,8 +83,13 @@ "last_modified_at":{ "type": "string", "format": "YYYY-MM-DDTHH:mm:ss.sssZ" - } - , + }, + "catchments":{ + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "audit": { "type": ["null", "object"], "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-avni/source_avni/source.py b/airbyte-integrations/connectors/source-avni/source_avni/source.py index e12d819d2c8c4..e6c65ceadb7d3 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/source.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/source.py @@ -2,224 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import boto3 -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -class AvniStream(HttpStream, ABC): - url_base = "https://app.avniproject.org/api/" - primary_key = "ID" - cursor_value = None - current_page = 0 - last_record = None - - def __init__(self, start_date: str, auth_token: str, **kwargs): - super().__init__(**kwargs) - - self.start_date = start_date - self.auth_token = auth_token - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = {"lastModifiedDateTime": self.state["last_modified_at"]} - if next_page_token: - params.update(next_page_token) - return params - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - - return {"auth-token": self.auth_token} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - - data = response.json()["content"] - if data: - self.last_record = data[-1] - - yield from data - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[StreamData]: - - records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - for record in records: - last_modified_at = record["audit"]["Last modified at"] - record["last_modified_at"] = last_modified_at - yield record - - def update_state(self) -> None: - - if self.last_record: - updated_last_date = self.last_record["audit"]["Last modified at"] - if updated_last_date > self.state["last_modified_at"]: - self.state = {self.cursor_field: updated_last_date} - self.last_record = None - - return None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - - total_elements = int(response.json()["totalElements"]) - page_size = int(response.json()["pageSize"]) - - if total_elements == page_size: - self.current_page = self.current_page + 1 - return {"page": self.current_page} - - self.update_state() - - self.current_page = 0 - - return None - - -class IncrementalAvniStream(AvniStream, IncrementalMixin, ABC): - - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - return "last_modified_at" - - @property - def state(self) -> Mapping[str, Any]: - - if self.cursor_value: - return {self.cursor_field: self.cursor_value} - else: - return {self.cursor_field: self.start_date} - - @state.setter - def state(self, value: Mapping[str, Any]): - self.cursor_value = value[self.cursor_field] - self._state = value - - -class Subjects(IncrementalAvniStream): - - """ - This implement Subject Stream in Source Avni - Api docs : https://avni.readme.io/docs/api-guide - Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 - """ - - def path(self, **kwargs) -> str: - return "subjects" - - -class ProgramEnrolments(IncrementalAvniStream): - - """ - This implement ProgramEnrolments Stream in Source Avni - Api docs : https://avni.readme.io/docs/api-guide - Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 - """ - - def path(self, **kwargs) -> str: - return "programEnrolments" - - -class ProgramEncounters(IncrementalAvniStream): - - """ - This implement ProgramEncounters Stream in Source Avni - Api docs : https://avni.readme.io/docs/api-guide - Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 - """ - - def path(self, **kwargs) -> str: - return "programEncounters" - - -class Encounters(IncrementalAvniStream): - - """ - This implement Encounters Stream in Source Avni - Api docs : https://avni.readme.io/docs/api-guide - Api endpoints : https://app.swaggerhub.com/apis-docs/samanvay/avni-external/1.0.0 - """ - - def path(self, **kwargs) -> str: - return "encounters" - - -class SourceAvni(AbstractSource): - def get_client_id(self): - - url_client = "https://app.avniproject.org/idp-details" - response = requests.get(url_client) - response.raise_for_status() - client = response.json() - return client["cognito"]["clientId"] - - def get_token(self, username: str, password: str, app_client_id: str) -> str: - - """ - Avni Api Authentication : https://avni.readme.io/docs/api-guide#authentication - AWS Cognito for authentication : https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cognito-idp/client/initiate_auth.html - """ - - client = boto3.client("cognito-idp", region_name="ap-south-1") - response = client.initiate_auth( - ClientId=app_client_id, AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": username, "PASSWORD": password} - ) - return response["AuthenticationResult"]["IdToken"] - - def check_connection(self, logger, config) -> Tuple[bool, any]: - - username = config["username"] - password = config["password"] - - try: - client_id = self.get_client_id() - except Exception as error: - return False, str(error) + ": Please connect With Avni Team" - - try: - auth_token = self.get_token(username, password, client_id) - stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} - next(Subjects(**stream_kwargs).read_records(SyncMode.full_refresh)) - return True, None - - except Exception as error: - return False, error - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - - username = config["username"] - password = config["password"] - - try: - client_id = self.get_client_id() - except Exception as error: - print(str(error) + ": Please connect With Avni Team") - raise error - - auth_token = self.get_token(username, password, client_id) - - stream_kwargs = {"auth_token": auth_token, "start_date": config["start_date"]} - - return [ - Subjects(**stream_kwargs), - ProgramEnrolments(**stream_kwargs), - ProgramEncounters(**stream_kwargs), - Encounters(**stream_kwargs), - ] +# Declarative Source +class SourceAvni(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml b/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml deleted file mode 100644 index 2ecbcd34dae22..0000000000000 --- a/airbyte-integrations/connectors/source-avni/source_avni/spec.yaml +++ /dev/null @@ -1,26 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/avni -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Avni Spec - type: object - required: - - username - - password - - start_date - properties: - username: - type: string - description: Your Avni platform username - title: Username - password: - type: string - description: Your Avni platform password - title: Password - airbyte_secret: true - start_date: - type: string - default: "2000-06-23T01:30:00.000Z" - description: Specify Date and time from which you want to fetch data - title: Start date and time - examples: - - "2000-10-31T01:30:00.000Z" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py new file mode 100644 index 0000000000000..1f8467847f488 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_avni.components import CustomAuthenticator +from unittest.mock import Mock, patch + + +@patch('boto3.client') +def test_token_property(mock_boto3_client): + + mock_cognito_client = Mock() + mock_boto3_client.return_value = mock_cognito_client + + config= { "username": "example@gmail.com", "api_key": "api_key" } + source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + source._username = Mock() + source._username.eval.return_value = "test_username" + source._password = Mock() + source._password.eval.return_value = "test_password" + source.get_client_id = Mock() + source.get_client_id.return_value = "test_client_id" + + mock_cognito_client.initiate_auth.return_value = { + "AuthenticationResult": { + "IdToken": "test_id_token" + } + } + token = source.token + mock_boto3_client.assert_called_once_with("cognito-idp", region_name="ap-south-1") + mock_cognito_client.initiate_auth.assert_called_once_with( + ClientId="test_client_id", + AuthFlow="USER_PASSWORD_AUTH", + AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} + ) + assert token == "test_id_token" + +def test_get_client_id(mocker): + + config= { "username": "example@gmail.com", "api_key": "api_key" } + source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + client_id = source.get_client_id() + expected_length = 26 + assert len(client_id) == expected_length \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py deleted file mode 100644 index ccd09b924011b..0000000000000 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_avni.source import IncrementalAvniStream - - -@fixture -def patch_incremental_base_class(mocker): - - mocker.patch.object(IncrementalAvniStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalAvniStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalAvniStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - - stream = IncrementalAvniStream(start_date="",auth_token="") - expected_cursor_field = "last_modified_at" - assert stream.cursor_field == expected_cursor_field - - -def test_update_state(patch_incremental_base_class): - - stream = IncrementalAvniStream(start_date="",auth_token="") - stream.state = {"last_modified_at":"2000-06-27T04:18:36.914Z"} - stream.last_record = {"audit":{"Last modified at":"2001-06-27T04:18:36.914Z"}} - expected_state = {"last_modified_at":"2001-06-27T04:18:36.914Z"} - stream.update_state() - assert stream.state == expected_state - - -def test_stream_slices(patch_incremental_base_class): - - stream = IncrementalAvniStream(start_date="",auth_token="") - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - - mocker.patch.object(IncrementalAvniStream, "cursor_field", "dummy_field") - stream = IncrementalAvniStream(start_date="",auth_token="") - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - - stream = IncrementalAvniStream(start_date="",auth_token="") - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - - stream = IncrementalAvniStream(start_date="",auth_token="") - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py deleted file mode 100644 index 7022f4de4ce4d..0000000000000 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_source.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import patch - -from source_avni.source import SourceAvni - - -def test_check_connection_success(mocker): - with patch('source_avni.source.SourceAvni.get_client_id') as get_client_id_mock, \ - patch('source_avni.source.SourceAvni.get_token') as get_token_mock, \ - patch('source_avni.source.Subjects.read_records') as read_records_mock: - get_client_id_mock.return_value = "ClientID" - get_token_mock.return_value = "Token" - read_records_mock.return_value = iter(["record1", "record2"]) - source = SourceAvni() - config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} - result,msg = source.check_connection(None, config_mock) - assert result is True - - -def test_streams(mocker): - - mocker.patch('source_avni.source.SourceAvni.get_token').return_value = 'fake_token' - source = SourceAvni() - config_mock = {"username": "test_user", "password": "test_password", "start_date": "2000-06-27T04:18:36.914Z"} - streams = source.streams(config_mock) - excepted_outcome = 4 - assert len(streams) == excepted_outcome - - -def test_get_client_id(mocker): - - source = SourceAvni() - client_id = source.get_client_id() - expected_length = 26 - assert len(client_id) == expected_length diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py deleted file mode 100644 index c0be8d8359d45..0000000000000 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_streams.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_avni.source import AvniStream - - -@pytest.fixture -def patch_base_class(mocker): - - mocker.patch.object(AvniStream, "path", "v0/example_endpoint") - mocker.patch.object(AvniStream, "primary_key", "test_primary_key") - mocker.patch.object(AvniStream, "__abstractmethods__", set()) - - -def test_request_params(mocker,patch_base_class): - - stream = AvniStream(start_date="",auth_token="") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page":5}} - stream.state = {"last_modified_at":"AnyDate"} - expected_params = {"lastModifiedDateTime":"AnyDate","page":5} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - - stream = AvniStream(start_date="",auth_token="") - response_mock = MagicMock() - response_mock.json.return_value = { - "totalElements": 20, - "totalPages": 10, - "pageSize": 20 - } - - stream.current_page = 1 - inputs = {"response": response_mock} - expected_token = {"page": 2} - - assert stream.next_page_token(**inputs) == expected_token - assert stream.current_page == 2 - - -def test_parse_response(patch_base_class,mocker): - - stream = AvniStream(start_date="",auth_token="") - response = MagicMock - response.content = b'{"content": [{"id": 1, "name": "John"}, {"id": 2, "name": "Jane"}]}' - - inputs = {"response": mocker.Mock(json=mocker.Mock(return_value={"content": [{"id": 1, "name": "Avni"}, {"id": 2, "name": "Airbyte"}]}))} - gen = stream.parse_response(**inputs) - assert next(gen) == {"id": 1, "name": "Avni"} - assert next(gen) == {"id": 2, "name": "Airbyte"} - - -def test_request_headers(patch_base_class): - - stream = AvniStream(start_date="",auth_token="") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - stream.auth_token = "Token" - expected_headers = {"auth-token":"Token"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - - stream = AvniStream(start_date="",auth_token="") - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - - response_mock = MagicMock() - response_mock.status_code = http_status - stream = AvniStream(start_date="",auth_token="") - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - - response_mock = MagicMock() - stream = AvniStream(start_date="",auth_token="") - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time From c8a6a6250c7feecc78b0b3f4785750437c112770 Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Thu, 7 Sep 2023 14:24:35 +0530 Subject: [PATCH 05/14] acceptance tests --- .../source-avni/acceptance-test-config.yml | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml index e21280f168e43..54ec3ab7e71f8 100644 --- a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml @@ -2,30 +2,30 @@ # for more information about how to configure these tests connector_image: airbyte/source-avni:dev acceptance_tests: - # spec: - # tests: - # - spec_path: "source_avni/spec.yaml" - # connection: - # tests: - # - config_path: "secrets/config.json" - # status: "succeed" - # - config_path: "integration_tests/invalid_config.json" - # status: "failed" - # discovery: - # tests: - # - config_path: "secrets/config.json" + spec: + tests: + - spec_path: "source_avni/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # incremental: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state: - # future_state_path: "integration_tests/abnormal_state.json" - # full_refresh: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" From 4c07377d55d359090fa7dcd60bbdb161fa66f197 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 15 Sep 2023 13:56:20 +0530 Subject: [PATCH 06/14] added the title fields --- .../connectors/source-avni/source_avni/manifest.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index d62154bead30d..1fef4a09dee3d 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -129,13 +129,16 @@ spec: username: type: string description: Your avni platform Username + title: Username password: type: string description: Your avni platform password + title: Password airbyte_secret: true start_date: type: string default: "2000-06-23T01:30:00.000Z" description: Specify Date and time from which you want to fetch data + title: Start Date examples: - "2000-10-31T01:30:00.000Z" From 3f01a9e1e5ccf69a5d6d2b2b8fea80b89fceb442 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Sat, 13 Apr 2024 15:52:44 +0530 Subject: [PATCH 07/14] require user to provide avni base url --- airbyte-integrations/connectors/source-avni/Dockerfile | 2 +- airbyte-integrations/connectors/source-avni/metadata.yaml | 2 +- .../connectors/source-avni/source_avni/components.py | 7 +++---- .../connectors/source-avni/source_avni/manifest.yaml | 8 +++++++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile index 09b5073bfc31c..cb3759ec2e5e9 100644 --- a/airbyte-integrations/connectors/source-avni/Dockerfile +++ b/airbyte-integrations/connectors/source-avni/Dockerfile @@ -34,5 +34,5 @@ COPY source_avni ./source_avni ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-avni diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml index 79cdb44b311a7..6c1d9b73f1bd1 100644 --- a/airbyte-integrations/connectors/source-avni/metadata.yaml +++ b/airbyte-integrations/connectors/source-avni/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5d297ac7-355e-4a04-be75-a5e7e175fc4e - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-avni githubIssueLabel: source-avni icon: avni.svg diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py index ba6d6eb99a6b5..1f7f7f3cf873b 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/components.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/components.py @@ -7,7 +7,6 @@ @dataclass class CustomAuthenticator(BasicHttpAuthenticator): - @property def token(self) -> str: @@ -28,9 +27,9 @@ def auth_header(self) -> str: return "auth-token" def get_client_id(self): - - url_client = "https://app.avniproject.org/idp-details" - response = requests.get(url_client) + + url_client = self.config["url_base"] + "/idp-details" + response = requests.get(url_client, timeout=30) response.raise_for_status() client = response.json() return client["cognito"]["clientId"] diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index 1fef4a09dee3d..b769efd251494 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -9,12 +9,13 @@ definitions: requester: type: HttpRequester - url_base: "https://app.avniproject.org/api" + url_base: "{{config['url_base']}}/api" http_method: "GET" authenticator: class_name: source_avni.components.CustomAuthenticator username: "{{config['username']}}" password: "{{config['password']}}" + idp_base: "{{config['url_base']}}" retriever: type: SimpleRetriever @@ -123,6 +124,7 @@ spec: required: - username - password + - url_base - start_date additionalProperties: true properties: @@ -135,6 +137,10 @@ spec: description: Your avni platform password title: Password airbyte_secret: true + url_base: + type: string + description: Your avni platform base url, with no trailing slash (/) + title: Base URL (no trailing /) start_date: type: string default: "2000-06-23T01:30:00.000Z" From 77740183a73e68b8cd7f970c06838c1ae35082cc Mon Sep 17 00:00:00 2001 From: Siddhant Singh Date: Thu, 9 May 2024 13:00:37 +0530 Subject: [PATCH 08/14] added the api for approval status --- .../connectors/source-avni/Dockerfile | 19 ++++---- .../source-avni/source_avni/manifest.yaml | 15 +++++- .../source_avni/schemas/approvalStatuses.json | 46 +++++++++++++++++++ 3 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile index cb3759ec2e5e9..7cabf21c80e55 100644 --- a/airbyte-integrations/connectors/source-avni/Dockerfile +++ b/airbyte-integrations/connectors/source-avni/Dockerfile @@ -1,33 +1,32 @@ FROM python:3.9.11-alpine3.15 as base -# build and load all requirements +# Build and load all requirements FROM base as builder WORKDIR /airbyte/integration_code -# upgrade pip to the latest version +# Upgrade pip to the latest version and install build dependencies RUN apk --no-cache upgrade \ && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - + && apk --no-cache add tzdata build-base libffi-dev openssl-dev COPY setup.py ./ -# install necessary packages to a temporary folder +# Install necessary packages to a temporary folder RUN pip install --prefix=/install . -# build a clean environment +# Build a clean environment FROM base WORKDIR /airbyte/integration_code -# copy all loaded and built libraries to a pure basic image +# Copy all loaded and built libraries to a pure basic image COPY --from=builder /install /usr/local -# add default timezone settings +# Add default timezone settings COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone -# bash is installed for more convenient debugging. +# Bash is installed for more convenient debugging. RUN apk --no-cache add bash -# copy payload code only +# Copy payload code only COPY main.py ./ COPY source_avni ./source_avni diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index b769efd251494..2928117a1e075 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -29,7 +29,7 @@ definitions: field_name: "size" pagination_strategy: type: "PageIncrement" - page_size: 100 + page_size: 20 page_token_option: type: "RequestOption" inject_into: "request_parameter" @@ -103,18 +103,29 @@ definitions: $parameters: path: "/encounters" + approval_status_stream: + $ref: "#/definitions/base_stream" + name: "approvalStatuses" + primary_key: "Entity ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/approvalStatuses" + streams: - "#/definitions/subjects_stream" - "#/definitions/program_enrolments_stream" - "#/definitions/program_encounters_stream" - "#/definitions/encounters_stream" + - "#/definitions/approval_status_stream" check: type: CheckStream stream_names: - "subjects" - spec: type: Spec documentation_url: https://docs.airbyte.com/integrations/sources/avni diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json new file mode 100644 index 0000000000000..e8e857f8ba56b --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "Entity ID": { + "type": "string" + }, + "Entity type": { + "type": "string" + }, + "Entity type ID": { + "type": "string" + }, + "Approval status": { + "type": "string" + }, + "Approval status comment": { + "type": ["null", "string"] + }, + "Status date time": { + "type": "string", + "format": "date-time" + }, + "audit": { + "type": "object", + "properties": { + "Created at": { + "type": "string", + "format": "date-time" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": "string" + }, + "Last modified by": { + "type": "string" + } + } + } + } + } + \ No newline at end of file From afe35a567bf51b30254ffb66bb064494af866aaa Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 10 May 2024 10:51:39 +0530 Subject: [PATCH 09/14] added last_modified_at --- .../source-avni/source_avni/schemas/approvalStatuses.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json index e8e857f8ba56b..f5241fe241d46 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json @@ -22,6 +22,10 @@ "type": "string", "format": "date-time" }, + "last_modified_at": { + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, "audit": { "type": "object", "properties": { @@ -43,4 +47,4 @@ } } } - \ No newline at end of file + From b599facb0c2c03b6b1a87304c9d5964652f16344 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 10 May 2024 10:51:43 +0530 Subject: [PATCH 10/14] added pytz --- airbyte-integrations/connectors/source-avni/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-avni/setup.py b/airbyte-integrations/connectors/source-avni/setup.py index b5f12bb51f556..4aead2c068a7d 100644 --- a/airbyte-integrations/connectors/source-avni/setup.py +++ b/airbyte-integrations/connectors/source-avni/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "boto3==1.18.0", + "pytz==2024.1", ] TEST_REQUIREMENTS = [ From 6e278fd3e50b8fc1cd1516a70430a81d69c778d4 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Fri, 10 May 2024 17:24:39 +0530 Subject: [PATCH 11/14] page size = 1000 specify order in config properties --- .../connectors/source-avni/source_avni/manifest.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index 2928117a1e075..a0cd35becc16b 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -29,7 +29,7 @@ definitions: field_name: "size" pagination_strategy: type: "PageIncrement" - page_size: 20 + page_size: 1000 page_token_option: type: "RequestOption" inject_into: "request_parameter" @@ -143,19 +143,23 @@ spec: type: string description: Your avni platform Username title: Username + order: 1 password: type: string description: Your avni platform password title: Password airbyte_secret: true + order: 2 url_base: type: string description: Your avni platform base url, with no trailing slash (/) title: Base URL (no trailing /) + order: 3 start_date: type: string default: "2000-06-23T01:30:00.000Z" description: Specify Date and time from which you want to fetch data title: Start Date + order: 4 examples: - "2000-10-31T01:30:00.000Z" From 56f6f5271f20dcacd191755d92a12afd8d41fd86 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Wed, 29 May 2024 09:10:58 +0530 Subject: [PATCH 12/14] locations stream --- .../connectors/source-avni/source_avni/manifest.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index 4d53b576e93d6..2b58b6686df6c 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -114,12 +114,24 @@ definitions: $parameters: path: "/approvalStatuses" + locations_stream: + $ref: "#/definitions/base_stream" + name: "locations" + primary_key: "Entity ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/locations" + streams: - "#/definitions/subjects_stream" - "#/definitions/program_enrolments_stream" - "#/definitions/program_encounters_stream" - "#/definitions/encounters_stream" - "#/definitions/approval_status_stream" + - "#/definitions/locations_stream" check: type: CheckStream From 199f77292d829744fd402a1720c3a5a7274387d9 Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Thu, 30 May 2024 15:24:17 +0530 Subject: [PATCH 13/14] added schema for locations also locations has ID not Entity ID --- .../source-avni/source_avni/manifest.yaml | 2 +- .../source_avni/schemas/locations.json | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index 2b58b6686df6c..765f0fee5b05e 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -117,7 +117,7 @@ definitions: locations_stream: $ref: "#/definitions/base_stream" name: "locations" - primary_key: "Entity ID" + primary_key: "ID" incremental_sync: $ref: "#/definitions/incremental_base" transformations: diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json new file mode 100644 index 0000000000000..2abf485be7986 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": "string" + }, + "External ID": { + "type": [ + "null", + "string" + ] + }, + "Title": { + "type": "string" + }, + "Type": { + "type": "string" + }, + "Level": { + "type": "number" + }, + "Voided": { + "type": "boolean" + }, + "customProperties": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + }, + "last_modified_at": { + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "audit": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "Created at": { + "type": [ + "null", + "string" + ], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": [ + "null", + "string" + ], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": [ + "null", + "string" + ] + }, + "Last modified by": { + "type": [ + "null", + "string" + ] + } + } + } + } +} \ No newline at end of file From 8211a6b21ffd323d24633e16c162f5effa9a6e1c Mon Sep 17 00:00:00 2001 From: Rohit Chatterjee Date: Mon, 3 Jun 2024 14:31:41 +0530 Subject: [PATCH 14/14] added Parent --- .../source-avni/source_avni/schemas/locations.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json index 2abf485be7986..354076ab700bf 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json @@ -35,6 +35,13 @@ "type": "string", "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, + "Parent": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + }, "audit": { "type": [ "null",