s3 presigned links

Author: dantemazza

.gitignore

@@ -1,10 +1,6 @@
 *venv*
 __pycache__/
 *.pyc
-<<<<<<< HEAD
 .vscode
 **/.env
 tmp
-=======
-.vscode
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66

servers/tenant/blueprints/event_driven/ticket.py

@@ -4,42 +4,25 @@
 
 from numpy import number
 from flask import make_response, request, jsonify, Blueprint
-<<<<<<< HEAD
-=======
-from flask_cors import cross_origin
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 import sys
 
 sys.path.insert(0, "..")  # import parent folder
 
-<<<<<<< HEAD
 from controllers.controllerMapper import TicketController, TicketStatusController
-=======
-from controllers.controllerMapper import TicketController
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 from models.models import TicketEvents
 from utils import (
     AlchemyEncoder,
     alchemyConverter,
 )
 
-<<<<<<< HEAD
 # from flask_cognito_lib.decorators import auth_required
 
 ticket_bp = Blueprint("ticket_bp", __name__, url_prefix="ticket")
 
 ticket_controller = TicketController()
 ticket_status_controller = TicketStatusController()
 PIECES_SEPERATOR = ",+-"
-=======
-ticket_bp = Blueprint("ticket_bp", __name__, url_prefix="ticket")
-
-# TODO: USER BASED AUTH
-
-ticket_controller = TicketController()
-
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 """
 Route expects requests of format:
 
@@ -97,12 +80,7 @@ def ticket_get_all_with_status(status):  # create ticket
 
 
 @ticket_bp.route("/", methods=["POST"])
-<<<<<<< HEAD
 #@auth_required()
-=======
-@cross_origin(supports_credentials=True)
-@require_appkey
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 def ticket_post():  # create ticket
     print("Creating ticket from the following JSON:")
     print(request.data)
@@ -132,7 +110,6 @@ def ticket_edit(ticket_id):  # create ticket
     ticket_dict["pieces"] =  PIECES_SEPERATOR.join(ticket_dict["pieces"])
     ticket_event = ticket_controller._create_base_event(ticket_dict)
 
-<<<<<<< HEAD
     response = {"ticketId": ticket_event.ticketId}
     return make_response(json.dumps(response))
 
@@ -150,46 +127,19 @@ def get_clean_filters_dict(immutable_args):
     if "limit" in sql_filters:
         del sql_filters["limit"]
     return sql_filters
-=======
-    return {"success"}
-
-
-# http://127.0.0.1:6767/api/ticket/?start=2022-01-01T00:00:00&end=2022-04-04T00:00:00&shipperName=Eric%20Shea
-# curl http://127.0.0.1:6767/api/ticket/?shipperName
-# # curl http://127.0.0.1:6767/api/ticket?key=a
-# # curl http://127.0.0.1:6767/api/ticket/?start=2022-01-01T00:00:00Z&end=2022-04-04T00:00:00Z
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 def corsify(resp):
     resp = make_response(json.dumps(resp))
     resp.headers['Access-Control-Allow-Origin'] = '*'
     resp.headers['Access-Control-Allow-Headers'] = ['Origin', 'X-Requested-With', 'Content-Type', 'Accept']
     return resp
 
-<<<<<<< HEAD
 def validate_date_format(date_text):
     try:
         return datetime.strptime(date_text, "%Y-%m-%dT%H:%M:%S")
     except ValueError:
         raise ValueError("Incorrect data format, should be %Y-%m-%dT%H:%M:%S")
 
-=======
-def get_clean_filters_dict(immutable_args):
-    sql_filters = dict(immutable_args)
-    if "start" in sql_filters: 
-        del sql_filters["start"]
-    if "end" in sql_filters:
-        del sql_filters["end"]
-    if "limit" in sql_filters:
-        del sql_filters["limit"]
-    return sql_filters
-
-def validate_date_format(date_text):
-    try:
-        return datetime.strptime(date_text, "%Y-%m-%dT%H:%M:%S")
-    except ValueError:
-        raise ValueError("Incorrect data format, should be %Y-%m-%dT%H:%M:%S")
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 def default_start():
     dt_start = validate_date_format("1900-01-01T00:00:00")
@@ -199,7 +149,6 @@ def default_end():
     dt_end = validate_date_format("2100-01-01T00:00:00")
     return dt_end
 
-<<<<<<< HEAD
 def default_end():
     dt_end = validate_date_format("2100-01-01T00:00:00")
     return dt_end
@@ -227,45 +176,9 @@ def ticket_get_all():
     for ticket in res:
         ticket["pieces"] = ticket["pieces"].split(PIECES_SEPERATOR)
         ticket["ticketStatus"]["currentStatus"] = ticket["ticketStatus"]["currentStatus"].value
-=======
-@ticket_bp.route("/", methods=["GET"])
-@cross_origin(supports_credentials=True)
-# @require_appkey
-def ticket_get_all():
-    filters = request.args or {}
-    sql_filters = get_clean_filters_dict(filters)
-    limit = 5000 if "limit" not in filters else filters["limit"]
-
-    dt_start = validate_date_format(filters["start"]) if "start" in filters else default_start()
-    dt_end = validate_date_format(filters["end"]) if "end" in filters else default_end()
-
-    data = ticket_controller._get_latest_event_objects_in_range(dt_start, dt_end, sql_filters, number_of_res=limit)
-    
-    res = alchemyConverter(data)
-    
-    return corsify(res)
-
-
-@ticket_bp.route("/<ticket_id>", methods=["GET"])
-@cross_origin(supports_credentials=True)
-# @require_appkey
-def ticket_get(ticket_id):
-    filters = request.args.get("filters") or {}
-    
-    
-    sql_filters = get_clean_filters_dict(filters)
-    sql_filters["ticketId"] = ticket_id
-    data = ticket_controller._get_latest_event_objects_in_range(
-        default_start(), default_end(), filters=sql_filters
-    )
-
-    res = alchemyConverter(data[0])
-    return corsify(res)
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
     return make_response(json.dumps(res, cls=AlchemyEncoder))
 
-<<<<<<< HEAD
 
 def get_single(ticket_id):
     filters = request.args.get("filters") or {}
@@ -284,38 +197,10 @@ def ticket_get(ticket_id):
     data = get_single(ticket_id)
     res = alchemyConverter(data)
     return make_response(json.dumps(res, cls=AlchemyEncoder))
-=======
-{
-    "datetime" : "value", 
-    "filters" : {
-        "field1": "value1",
-        "field2": "value2",
-        ....
-    }
-}
-
-"""
-
-
-
-# @ticket_bp.route("/attribute/{attribute_name}", methods=["GET"])
-# @require_appkey
-# def ticket_attribute_get(attribute_name):
-
-#     filters.extend({"ticket_id": ticket_id})
-
-#     latest_ticket = ticket_controller._get_latest_event_objects(
-#         number_of_res=number_of_res, filters=filters
-#     )
-
-#     res = alchemyConverter(latest_ticket)
-#     response = json.dumps(res, cls=AlchemyEncoder)
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 #     return response
 
-
-<<<<<<< HEAD
+"""
 {
     "datetime" : "value", 
     "filters" : {
@@ -326,9 +211,6 @@ def ticket_get(ticket_id):
 }
 
 """
-=======
-
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 """
 Route expects requests of format:
@@ -348,29 +230,3 @@ def ticket_get(ticket_id):
 
 """
 
-<<<<<<< HEAD
-=======
-
-@ticket_bp.route("/<ticket_id>", methods=["PUT"])
-@cross_origin(supports_credentials=True)
-@require_appkey
-def ticket_update(ticket_id):
-
-    update_dict = request.form["update_dict"]
-
-    # remove ticketId and ticketEventId if present
-    update_dict.pop(ticket_controller.primary_key, None)
-    update_dict.pop(TicketEvents.non_prim_identifying_column_name, None)
-
-    filters = request.form["filters"]
-    filters.extend({"ticket_id": ticket_id})
-
-    updated_object = ticket_controller._modify_latest_object(
-        update_dict, filters=filters
-    )
-
-    res = alchemyConverter(updated_object)
-    response = json.dumps(res, cls=AlchemyEncoder)
-
-    return response
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66

servers/tenant/blueprints/simple/users.py

@@ -32,11 +32,7 @@ def user_post():  # create ticket
 
 
 @user_bp.route("", methods=["PUT"])
-<<<<<<< HEAD
 #@auth_required()
-=======
-@require_appkey
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 def user_modify():
 
     userId = request.form["userId"]

servers/tenant/celery_client.py

@@ -4,20 +4,14 @@
 import io
 from uuid import uuid4
 import traceback
-<<<<<<< HEAD
 import PyPDF2
 import extraction.app as ex
 import extraction.extract as ext
-=======
-# import tenant.controllers.DocumentController as document_controller
-import PyPDF2
-import extraction.app as ex
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 from celery import group
 
 from tenant.controllers.DocumentController import DocumentController
 import boto3
-
+from botocore.client import Config
 
 # def get_file_s3():
 #     s3_client = boto3.client('s3')
@@ -28,12 +22,14 @@
 #     download_url = s3_client.generate_presigned_url(
 #         'get_object',
 #         Params={'Bucket': BUCKET, 'Key': OBJECT, 'ResponseContentDisposition': 'attachment'},
-#         ExpiresIn=600)
+#         ExpiresIn=3600)
 
 #     view_url = s3_client.generate_presigned_url(
 #         'get_object',
 #         Params={'Bucket': BUCKET, 'Key': OBJECT},
-#         ExpiresIn=600)
+#         ExpiresIn=3600)
+
+
 TENANT = "test-tenant1"
 BUCKET = f"{TENANT}-bucket"
 aws_access_key_id = os.getenv("aws_access_key_id")
@@ -44,24 +40,19 @@
 logger = get_logger(__name__)
 FAILURE = -1
 SUCCESS = 0
-<<<<<<< HEAD
 PIECES_SEPERATOR = ",+-"
 UPLOAD_FOLDER = "/opt/metadata-extraction/uploads"    
-s3 = boto3.resource('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+s3 = boto3.resource('s3', region_name='ca-central-1', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, config=Config(signature_version='s3v4'))
 bucket = s3.Bucket(BUCKET)
+s3_client = boto3.client('s3', region_name='ca-central-1', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, config=Config(signature_version='s3v4'))
 
 def fan_out(file, documentStatusId):
-=======
-UPLOAD_FOLDER = "/opt/metadata-extraction/uploads"    
-
-
-def fan_out(file):
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
     folder_uuid = uuid4()
     with io.BytesIO(file.read()) as open_pdf_file:
         read_pdf = PyPDF2.PdfFileReader(open_pdf_file)
         num_pages = read_pdf.getNumPages()
         folder = f"{UPLOAD_FOLDER}/{folder_uuid}"
+        
         os.mkdir(folder)
         for i in range(num_pages):
             output_pdf = PyPDF2.PdfFileWriter()
@@ -71,17 +62,11 @@ def fan_out(file):
             os.mkdir(f_dir)
             with open(f"{f_dir}/{file_uuid}.pdf", "wb") as f:
                 output_pdf.write(f)
-<<<<<<< HEAD
 
             bucket.upload_file(f"{f_dir}/{file_uuid}.pdf", f"documents/{folder_uuid}/{file_uuid}.pdf")
     file.close()
     pdf_folders = os.listdir(folder)
     return group([work.s(f"{folder}/{pdf_folder}", documentStatusId) for pdf_folder in pdf_folders])
-=======
-    file.close()
-    pdf_folders = os.listdir(folder)
-    return group([work.s(f"{folder}/{pdf_folder}") for pdf_folder in pdf_folders])
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
 
 
 def do_all_work(tasks_to_run):
@@ -90,13 +75,18 @@ def do_all_work(tasks_to_run):
 
 
 @client.task
-<<<<<<< HEAD
 def work(pdf_folder, documentStatusId):
     document_controller = DocumentController()
     pdf_file = f"{pdf_folder}.pdf"
+    OBJECT = f"documents{pdf_file.replace(UPLOAD_FOLDER, '')}"
+    view_url = s3_client.generate_presigned_url(
+        'get_object',
+        Params={'Bucket': BUCKET, 'Key': OBJECT},
+        ExpiresIn=3600)
     try:
         doclist = ex.work(pdf_folder)
-        doclist["orderS3Link"] = f"s3://{BUCKET}/documents/{pdf_file.replace(UPLOAD_FOLDER, '')}"
+        doclist["orderS3Path"] = f"s3://{BUCKET}/{OBJECT}"
+        doclist["orderS3Link"] = view_url
         doclist["pieces"] = PIECES_SEPERATOR.join(doclist["pieces"])
         doclist["documentStatusId"] = documentStatusId
         doclist["success"] = True
@@ -105,19 +95,11 @@ def work(pdf_folder, documentStatusId):
         logger.info(f"file {pdf_folder}/{pdf_file} error. msg: {str(e)}")
         logger.info(traceback.format_exc())
         doclist = ext.generate_doclist({})
-        doclist["orderS3Link"] = f"s3://{BUCKET}/documents/{pdf_file.replace(UPLOAD_FOLDER, '')}"
+        doclist["orderS3Path"] = f"s3://{BUCKET}/{OBJECT}"
+        doclist["orderS3Link"] = view_url
         doclist["pieces"] = PIECES_SEPERATOR.join(doclist["pieces"])
         doclist["documentStatusId"] = documentStatusId
         doclist["success"] = False
         document_controller._create(doclist)
-=======
-def work(pdf_folder):
-    pdf_file = f"{pdf_folder}.pdf"
-    try:
-        doclist = ex.work(pdf_folder)
-    except Exception as e:
-        logger.info(f"file {pdf_folder}/{pdf_file} error. msg: {str(e)}")
-        logger.info(traceback.format_exc())
->>>>>>> 32dee55d98864ba43414c8757ab4abe2e4881f66
         return {"status": FAILURE, "folder": pdf_folder}
     return {"status": SUCCESS, "folder": pdf_folder, "doclist": doclist}