From 6668c122b81189a7c9d8ff414428f7e9572e584d Mon Sep 17 00:00:00 2001 From: gavin-aguiar Date: Fri, 26 Jan 2024 15:58:29 -0600 Subject: [PATCH 1/5] Added new service bus batch tests --- __azurite_db_blob__.json | 1 + __azurite_db_blob_extent__.json | 1 + .../function_app.py | 22 +++++++++++++++++ tests/endtoend/test_servicebus_functions.py | 24 +++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 __azurite_db_blob__.json create mode 100644 __azurite_db_blob_extent__.json diff --git a/__azurite_db_blob__.json b/__azurite_db_blob__.json new file mode 100644 index 00000000..975ab98a --- /dev/null +++ b/__azurite_db_blob__.json @@ -0,0 +1 @@ +{"filename":"d:\\Repos\\azure-functions-python-worker\\__azurite_db_blob__.json","collections":[{"name":"$SERVICES_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{},"constraints":null,"uniqueNames":["accountName"],"transforms":{},"objType":"$SERVICES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$CONTAINERS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$CONTAINERS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$BLOBS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]},"snapshot":{"name":"snapshot","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOBS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$BLOCKS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"blobName":{"name":"blobName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOCKS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"persistenceMethod":"fs","autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} \ No newline at end of file diff --git a/__azurite_db_blob_extent__.json b/__azurite_db_blob_extent__.json new file mode 100644 index 00000000..3d24f615 --- /dev/null +++ b/__azurite_db_blob_extent__.json @@ -0,0 +1 @@ +{"filename":"d:\\Repos\\azure-functions-python-worker\\__azurite_db_blob_extent__.json","collections":[{"name":"$EXTENTS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"id":{"name":"id","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$EXTENTS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"persistenceMethod":"fs","autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} \ No newline at end of file diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py index e2114fd1..0c79271a 100644 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py +++ b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py @@ -53,3 +53,25 @@ def servicebus_trigger(msg: func.ServiceBusMessage) -> str: }) return result + + +@app.service_bus_queue_trigger( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue", + cardinality=func.Cardinality.MANY) +@app.blob_output(arg_name="$return", + path="python-worker-tests/servicebus-triggered_batch.txt", + connection="AzureWebJobsStorage") +def servicebus_triggered(msg: func.ServiceBusMessage) -> str: + return str(msg) + + +@app.route(route="servicebus_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/servicebus-triggered_batch.txt", + connection="AzureWebJobsStorage") +def get_servicebus_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') \ No newline at end of file diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py index f5b05a9b..8779479b 100644 --- a/tests/endtoend/test_servicebus_functions.py +++ b/tests/endtoend/test_servicebus_functions.py @@ -45,6 +45,30 @@ def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' / \ 'servicebus_functions_stein' + @testutils.retryable_test(3, 5) + def test_servicebus_batch(self): + data = str({"value": "2024-01-19T12:50:41.250940Z"}) + r = self.webhost.request('POST', 'put_message', + data=data) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + max_retries = 10 + for try_no in range(max_retries): + # wait for trigger to process the queue item + time.sleep(1) + + try: + r = self.webhost.request('GET', 'servicebus_triggered') + self.assertEqual(r.status_code, 200) + msg = r.json() + self.assertEqual(msg['body'], data) + except (AssertionError, json.JSONDecodeError): + if try_no == max_retries - 1: + raise + else: + break + class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): From 44cff9a7c91ba334323fc5e656c841507397f71f Mon Sep 17 00:00:00 2001 From: gavin-aguiar Date: Tue, 30 Jan 2024 12:16:47 -0600 Subject: [PATCH 2/5] Added Servicebus batch e2e test --- .../function_app.py | 50 +++++++++++++++---- tests/endtoend/test_servicebus_functions.py | 23 +++------ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py index 0c79271a..909c70d5 100644 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py +++ b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py @@ -55,23 +55,53 @@ def servicebus_trigger(msg: func.ServiceBusMessage) -> str: return result +@app.route(route="put_message_batch") +@app.service_bus_queue_output( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueuebatch") +def put_message_batch(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body().decode('utf-8')) + return 'OK' + + @app.service_bus_queue_trigger( arg_name="msg", connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue", - cardinality=func.Cardinality.MANY) + queue_name="testqueuebatch", cardinality="many") @app.blob_output(arg_name="$return", - path="python-worker-tests/servicebus-triggered_batch.txt", + path="python-worker-tests/test-servicebus-batch.txt", connection="AzureWebJobsStorage") -def servicebus_triggered(msg: func.ServiceBusMessage) -> str: - return str(msg) +def servicebus_trigger_batch(msg: func.ServiceBusMessage) -> str: + msg = msg[0] + print(f"Message ============> {msg}") + result = json.dumps({ + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + }) + + return result -@app.route(route="servicebus_triggered") +@app.route(route="get_servicebus_triggered_batch") @app.blob_input(arg_name="file", - path="python-worker-tests/servicebus-triggered_batch.txt", + path="python-worker-tests/test-servicebus-batch.txt", connection="AzureWebJobsStorage") -def get_servicebus_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: +def get_servicebus_triggered_batch(req: func.HttpRequest, + file: func.InputStream) -> str: return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') \ No newline at end of file + file.read().decode('utf-8'), mimetype='application/json') + diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py index 8779479b..9d56f4f9 100644 --- a/tests/endtoend/test_servicebus_functions.py +++ b/tests/endtoend/test_servicebus_functions.py @@ -47,27 +47,18 @@ def get_script_dir(cls): @testutils.retryable_test(3, 5) def test_servicebus_batch(self): - data = str({"value": "2024-01-19T12:50:41.250940Z"}) - r = self.webhost.request('POST', 'put_message', + data = '{"value": "2024-01-19T12:50:41.250940Z"}' + r = self.webhost.request('POST', 'put_message_batch', data=data) self.assertEqual(r.status_code, 200) self.assertEqual(r.text, 'OK') - max_retries = 10 - for try_no in range(max_retries): - # wait for trigger to process the queue item - time.sleep(1) + time.sleep(2) - try: - r = self.webhost.request('GET', 'servicebus_triggered') - self.assertEqual(r.status_code, 200) - msg = r.json() - self.assertEqual(msg['body'], data) - except (AssertionError, json.JSONDecodeError): - if try_no == max_retries - 1: - raise - else: - break + r = self.webhost.request('GET', 'get_servicebus_triggered_batch') + self.assertEqual(r.status_code, 200) + msg = r.json() + self.assertEqual(msg["body"], data) class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): From 5b26ccd61a9d55ebf57c3f0e7702209a26b0e49d Mon Sep 17 00:00:00 2001 From: gavin-aguiar Date: Tue, 30 Jan 2024 12:18:53 -0600 Subject: [PATCH 3/5] Removed unwanted files --- __azurite_db_blob__.json | 1 - __azurite_db_blob_extent__.json | 1 - 2 files changed, 2 deletions(-) delete mode 100644 __azurite_db_blob__.json delete mode 100644 __azurite_db_blob_extent__.json diff --git a/__azurite_db_blob__.json b/__azurite_db_blob__.json deleted file mode 100644 index 975ab98a..00000000 --- a/__azurite_db_blob__.json +++ /dev/null @@ -1 +0,0 @@ -{"filename":"d:\\Repos\\azure-functions-python-worker\\__azurite_db_blob__.json","collections":[{"name":"$SERVICES_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{},"constraints":null,"uniqueNames":["accountName"],"transforms":{},"objType":"$SERVICES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$CONTAINERS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$CONTAINERS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$BLOBS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]},"snapshot":{"name":"snapshot","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOBS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]},{"name":"$BLOCKS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"blobName":{"name":"blobName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOCKS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"persistenceMethod":"fs","autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} \ No newline at end of file diff --git a/__azurite_db_blob_extent__.json b/__azurite_db_blob_extent__.json deleted file mode 100644 index 3d24f615..00000000 --- a/__azurite_db_blob_extent__.json +++ /dev/null @@ -1 +0,0 @@ -{"filename":"d:\\Repos\\azure-functions-python-worker\\__azurite_db_blob_extent__.json","collections":[{"name":"$EXTENTS_COLLECTION$","data":[],"idIndex":null,"binaryIndices":{"id":{"name":"id","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$EXTENTS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"disableFreeze":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[],"dirtyIds":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"persistenceMethod":"fs","autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} \ No newline at end of file From eb00ccd9d78befe83cfa1e5719740607589a064f Mon Sep 17 00:00:00 2001 From: gavin-aguiar Date: Tue, 30 Jan 2024 12:31:50 -0600 Subject: [PATCH 4/5] Fixed flake8 validation --- .../servicebus_functions_stein/function_app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py index 909c70d5..c004cd4b 100644 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py +++ b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py @@ -104,4 +104,3 @@ def get_servicebus_triggered_batch(req: func.HttpRequest, file: func.InputStream) -> str: return func.HttpResponse( file.read().decode('utf-8'), mimetype='application/json') - From 6c7960b98bf9223c360bf61ed140c2bf396e1630 Mon Sep 17 00:00:00 2001 From: gavin-aguiar Date: Thu, 1 Feb 2024 13:22:20 -0600 Subject: [PATCH 5/5] Updated data passed to SB --- tests/endtoend/test_servicebus_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py index 9d56f4f9..f947017b 100644 --- a/tests/endtoend/test_servicebus_functions.py +++ b/tests/endtoend/test_servicebus_functions.py @@ -47,7 +47,7 @@ def get_script_dir(cls): @testutils.retryable_test(3, 5) def test_servicebus_batch(self): - data = '{"value": "2024-01-19T12:50:41.250940Z"}' + data = '{"value": "2024-01-19T12:50:41.250941Z"}' r = self.webhost.request('POST', 'put_message_batch', data=data) self.assertEqual(r.status_code, 200)