From 914465fe1902755794f293b99fde7ca50576e179 Mon Sep 17 00:00:00 2001 From: anilmahajan Date: Thu, 3 Oct 2024 15:08:20 +0530 Subject: [PATCH] new upsert --- .../features/salesforcesink/RunTime.feature | 38 +++++++++++++++++++ .../BigQuery/BigQueryCreateTableQuery.txt | 4 +- .../BigQuery/BigQueryInsertDataQuery.txt | 8 ++-- .../resources/pluginParameters.properties | 2 +- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/e2e-test/features/salesforcesink/RunTime.feature b/src/e2e-test/features/salesforcesink/RunTime.feature index 90fc4938..1d85b554 100644 --- a/src/e2e-test/features/salesforcesink/RunTime.feature +++ b/src/e2e-test/features/salesforcesink/RunTime.feature @@ -132,3 +132,41 @@ Feature: Salesforce Sink - Run time Scenarios Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate the values of records transferred from Bigquery to Salesforce is equal + + @SINK-TS-SF-RNTM-04 @BQ_SOURCE_TEST @DELETE_TEST_DATA + Scenario: Verify user should be able to see ingest the records successfully using upsert operation + When Open Datafusion Project to configure pipeline + And Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + And Select Sink plugin: "Salesforce" from the plugins list + And Connect plugins: "BigQuery" and "Salesforce" to establish connection + And Navigate to the properties page of plugin: "Salesforce" + And fill Authentication properties for Salesforce Admin user + Then Enter input plugin property: "referenceName" with value: "ReferenceName" + And Select radio button plugin property: "operation" with value: "upsert" + Then Enter input plugin property: "externalIdField" with value: "UpsertColumnvalue" + And Enter input plugin property: "sObject" with value: "sobject.account" + And Select dropdown plugin property: "errorHandling" with option value: "Skip on error" + Then Validate "Salesforce" plugin properties + And Close the Plugin Properties page + And Save the pipeline + And Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred from Bigquery to Salesforce is equal diff --git a/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt b/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt index 17d217bd..6a836349 100644 --- a/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt +++ b/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt @@ -1,3 +1 @@ -create table `DATASET.TABLE_NAME` (Name STRING, Col_Timestamp__c TIMESTAMP, Col_Date__c DATE, Col_Currency__c FLOAT64, - Col_Email__c STRING, Col_Number__c FLOAT64, Col_GeoLocation__Latitude__s FLOAT64, -Col_GeoLocation__Longitude__s FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING) +create table DATASET.TABLE_NAME (Id__c FLOAT64, Name STRING, Col_Timestamp__c TIMESTAMP, Col_Date__c DATE, Col_Currency__c FLOAT64, Col_Email__c STRING, Col_Number__c FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING) diff --git a/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt b/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt index 6dd90956..e8d3a8c7 100644 --- a/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt +++ b/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt @@ -1,5 +1,3 @@ -insert into `DATASET.TABLE_NAME` (Name, Col_Timestamp__c, Col_Date__c, Col_Currency__c, Col_Email__c, Col_Number__c, -Col_GeoLocation__Latitude__s, Col_GeoLocation__Longitude__s, Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values -('adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',898365444,37.794116,-122.3432, -'984746334','abc/123','20:26:34','find'); - +insert into DATASET.TABLE_NAME (Id__c, Name, Col_Timestamp__c, Col_Date__c, Col_Currency__c, Col_Email__c, Col_Number__c,Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values +(786777,'adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',-122.3432, +'984746334','abc/123','20:26:34','find'); \ No newline at end of file diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 625132d1..b5a3a61f 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -18,7 +18,7 @@ invalid.admin.consumer.secret=lmnop891011 #SOQL Query simple.query=SELECT Id, Name, Phone FROM Account test.query=SELECT Id,Name,Col_Timestamp__c,Col_Date__c,Col_Currency__c,Col_Email__c,Col_Number__c,\ - Col_GeoLocation__Latitude__s,Col_GeoLocation__Longitude__s,Col__c,Col_Url__c,Col_Time__c,Col_Text__c FROM Automation_custom__c + Col__c,Col_Url__c,Col_Time__c,Col_Text__c FROM Automation_custom__c where.query=SELECT name FROM Opportunity WHERE StageName='Needs Analysis' groupby.query=SELECT CampaignId, AVG(Amount) FROM Opportunity GROUP BY CampaignId childtoparent.query=SELECT Id, Name, Account.Name FROM Contact WHERE Account.Industry = 'Chemicals'