diff --git a/.gitignore b/.gitignore index 0a37f3c..eb713bd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ __pycache__/ *.pyc .vscode **/.env -tmp \ No newline at end of file +tmp diff --git a/postgres/docker-compose.yml b/postgres/docker-compose.yml index 41d3013..2a6c2b6 100644 --- a/postgres/docker-compose.yml +++ b/postgres/docker-compose.yml @@ -5,8 +5,6 @@ services: image: postgres:latest ports: - 5432:5432 - env_file: - - env/postgres.env environment: POSTGRES_DB: tenant_db POSTGRES_HOST_AUTH_METHOD: trust diff --git a/servers/app.Dockerfile b/servers/app.Dockerfile new file mode 100644 index 0000000..03cecd4 --- /dev/null +++ b/servers/app.Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9 +RUN apt-get update && apt-get -y install qpdf poppler-utils && apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev +RUN apt -y install libpq-dev +COPY tenant/requirements.txt . +RUN pip3 install --upgrade pip +RUN pip3 install -r requirements.txt +RUN pip3 install psycopg2 +RUN git -C /root clone https://github.com/ShipSolver/flask-cognito-lib.git +RUN pip3 install -e /root/flask-cognito-lib +WORKDIR /opt/metadata-extraction +ENV PYTHONPATH . +EXPOSE 6767 +ENV aws_secret_access_key Mwi2Sq90taDAkUZwtiEuLHvTXZLzXDQZExPh53R4 +ENV aws_access_key_id AKIASPMMHOET3PNSICG4 +ENV AWS_REGION="us-east-1" +ENV AWS_COGNITO_USER_POOL_ID="us-east-1_6AUY6LKPZ" +ENV AWS_COGNITO_USER_POOL_CLIENT_ID="2vukbtukva3u0oh29lf32ghmkp" +ENV AWS_COGNITO_DOMAIN="https://shipsolver-dev.auth.us-east-1.amazoncognito.com" \ No newline at end of file diff --git a/servers/celery.Dockerfile b/servers/celery.Dockerfile new file mode 100644 index 0000000..78c3e35 --- /dev/null +++ b/servers/celery.Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.9 +RUN apt-get update && apt-get -y install qpdf poppler-utils && apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev +RUN apt -y install tesseract-ocr && apt -y install libtesseract-dev +COPY tenant/requirements.txt . +RUN pip3 install --upgrade pip +RUN pip3 install -r requirements.txt +RUN pip3 install psycopg2-binary +RUN git -C /root clone https://github.com/ShipSolver/flask-cognito-lib.git +RUN pip3 install -e /root/flask-cognito-lib +WORKDIR /opt/metadata-extraction/tenant +ENV PYTHONPATH .. +ENV aws_secret_access_key Mwi2Sq90taDAkUZwtiEuLHvTXZLzXDQZExPh53R4 +ENV aws_access_key_id AKIASPMMHOET3PNSICG4 +ENV AWS_REGION="us-east-1" +ENV AWS_COGNITO_USER_POOL_ID="us-east-1_6AUY6LKPZ" +ENV AWS_COGNITO_USER_POOL_CLIENT_ID="2vukbtukva3u0oh29lf32ghmkp" +ENV AWS_COGNITO_DOMAIN="https://shipsolver-dev.auth.us-east-1.amazoncognito.com" \ No newline at end of file diff --git a/servers/tenant/docker-compose.yml b/servers/docker-compose.yml similarity index 86% rename from servers/tenant/docker-compose.yml rename to servers/docker-compose.yml index 4823ffb..1601a98 100644 --- a/servers/tenant/docker-compose.yml +++ b/servers/docker-compose.yml @@ -23,7 +23,7 @@ services: dockerfile: celery.Dockerfile volumes: - .:/opt/metadata-extraction - command: celery -A __init__.client worker --loglevel=info -f celery.logs -Ofair -c 2 + command: celery -A config.client worker --loglevel=info -f celery.logs -Ofair -c 2 tty: true app: hostname: app.wlp.com @@ -36,8 +36,8 @@ services: - .:/opt/metadata-extraction container_name: app01 ports: - - "5000:5000" - command: python3 server/__init__.py + - "6767:6767" + command: python3 tenant/server.py tty: true flower: hostname: flower.wlp.com diff --git a/extraction/app.py b/servers/extraction/app.py similarity index 67% rename from extraction/app.py rename to servers/extraction/app.py index 4501c6b..93e50c5 100644 --- a/extraction/app.py +++ b/servers/extraction/app.py @@ -1,10 +1,12 @@ import os -# from multilingual_pdf2text.pdf2text import PDF2Text -# from multilingual_pdf2text.models.document_model.document import Document -# import pdfplumber -# import extraction.extract as e +from multilingual_pdf2text.pdf2text import PDF2Text +from multilingual_pdf2text.models.document_model.document import Document +import pdfplumber +import extraction.extract as e import json +from celery.utils.log import get_logger +logger = get_logger(__name__) def read_pdfplumber(file_name): with pdfplumber.open(file_name) as pdf: @@ -26,8 +28,9 @@ def work(folder_path): ml_page_text = list(content)[0]["text"] pp_text = read_pdfplumber(pdf_file) - - extract_json = e.extract(ml_page_text, plumber_page=pp_text) + for i in range(14): + logger.info("WE HERE----------------") + extract_json = e.generate_doclist(e.extract(ml_page_text, plumber_page=pp_text)) with open(f"{folder_path}/{pdf_uuid}.json", "w") as f: json.dump(extract_json, f, indent=2) diff --git a/extraction/const.py b/servers/extraction/const.py similarity index 63% rename from extraction/const.py rename to servers/extraction/const.py index d477c86..ceb86da 100644 --- a/extraction/const.py +++ b/servers/extraction/const.py @@ -6,30 +6,33 @@ #doclist_keys -HOUSE_REF = "house_ref" -BARCODE = "barcode" -FIRST_PARTY = "first_party" -NUM_PCS = "num_pcs" -PCS = "pcs" +BARCODE = "barcodeNumber" +HOUSE_REF = "houseReferenceNumber" WEIGHT = "weight" +NUM_PCS = "claimedNumberOfPieces" +BOL_NUM = "BOLNumber" +SPECIAL_SERVICES = "specialServices" +SPECIAL_INSTRUCTIONS = "specialInstructions" +CONSIGNEE = "consignee" +SHIPPER = "shipper" +COMPANY = "Company" +NAME = "Name" +ADDRESS = "Address" +POSTAL_CODE = "PostalCode" +PHONE_NUMBER = "PhoneNumber" + +NO_SIGNATURE_REQUIRED = "noSignatureRequired" +TAILGATE_AUTHORIZED = "tailgateAuthorized" + +FIRST_PARTY = "customerName" + +PCS = "pieces" PKG = "pkg" -WT_LBS = "wt(lbs)" +WT_LBS = "weight" COMMODITY_DESCRIPTION = "commodity_description" DIMS_IN = "dims(in)" -BOL_NUM = "bol_num" -SPECIAL_SERVICES = "special_services" -SPECIAL_INSTRUCTIONS = "special_instructions" - -COMPANY = "company" -NAME = "name" -ADDRESS = "address" -POSTAL_CODE = "postal_code" -PHONE_NUMBER = "phone_number" - -CONSIGNEE = "consignee" -SHIPPER = "shipper" CEVA_SHIPPER_FIELDS = [COMPANY, ADDRESS] CEVA_CONSIGNEE_FIELDS = [NAME, ADDRESS] diff --git a/extraction/extract.py b/servers/extraction/extract.py similarity index 88% rename from extraction/extract.py rename to servers/extraction/extract.py index 491ffbd..beb2473 100644 --- a/extraction/extract.py +++ b/servers/extraction/extract.py @@ -271,25 +271,23 @@ def generate_doclist(_list): HOUSE_REF: _list[HOUSE_REF] if HOUSE_REF in _list else "", BARCODE: _list[BARCODE] if BARCODE in _list else "", PCS: _list[PCS] if PCS in _list else [], - NUM_PCS: _list[NUM_PCS] if NUM_PCS in _list else "", + NUM_PCS: _list[NUM_PCS] if NUM_PCS in _list else 0, WEIGHT: _list[WEIGHT] if WEIGHT in _list else "", BOL_NUM: _list[BOL_NUM] if BOL_NUM in _list else "", SPECIAL_SERVICES: _list[SPECIAL_SERVICES] if SPECIAL_SERVICES in _list else "", SPECIAL_INSTRUCTIONS: _list[SPECIAL_INSTRUCTIONS] if SPECIAL_INSTRUCTIONS in _list else "", - CONSIGNEE: { - COMPANY: _list[CONSIGNEE][COMPANY] if CONSIGNEE in _list and COMPANY in _list[CONSIGNEE] else "", - NAME: _list[CONSIGNEE][NAME] if CONSIGNEE in _list and NAME in _list[CONSIGNEE] else "", - ADDRESS: _list[CONSIGNEE][ADDRESS] if CONSIGNEE in _list and ADDRESS in _list[CONSIGNEE] else "", - POSTAL_CODE: _list[CONSIGNEE][POSTAL_CODE] if CONSIGNEE in _list and POSTAL_CODE in _list[CONSIGNEE] else "", - PHONE_NUMBER: _list[CONSIGNEE][PHONE_NUMBER] if CONSIGNEE in _list and PHONE_NUMBER in _list[CONSIGNEE] else "" - }, - SHIPPER: { - COMPANY: _list[SHIPPER][COMPANY] if SHIPPER in _list and COMPANY in _list[SHIPPER] else "", - NAME: _list[SHIPPER][NAME] if SHIPPER in _list and NAME in _list[SHIPPER] else "", - ADDRESS: _list[SHIPPER][ADDRESS] if SHIPPER in _list and ADDRESS in _list[SHIPPER] else "", - POSTAL_CODE: _list[SHIPPER][POSTAL_CODE] if SHIPPER in _list and POSTAL_CODE in _list[SHIPPER] else "", - PHONE_NUMBER: _list[SHIPPER][PHONE_NUMBER] if SHIPPER in _list and PHONE_NUMBER in _list[SHIPPER] else "" - } + CONSIGNEE+COMPANY: _list[CONSIGNEE][COMPANY] if CONSIGNEE in _list and COMPANY in _list[CONSIGNEE] else "", + CONSIGNEE+NAME: _list[CONSIGNEE][NAME] if CONSIGNEE in _list and NAME in _list[CONSIGNEE] else "", + CONSIGNEE+ADDRESS: _list[CONSIGNEE][ADDRESS] if CONSIGNEE in _list and ADDRESS in _list[CONSIGNEE] else "", + CONSIGNEE+POSTAL_CODE: _list[CONSIGNEE][POSTAL_CODE] if CONSIGNEE in _list and POSTAL_CODE in _list[CONSIGNEE] else "", + CONSIGNEE+PHONE_NUMBER: _list[CONSIGNEE][PHONE_NUMBER] if CONSIGNEE in _list and PHONE_NUMBER in _list[CONSIGNEE] else "", + SHIPPER+COMPANY: _list[SHIPPER][COMPANY] if SHIPPER in _list and COMPANY in _list[SHIPPER] else "", + SHIPPER+NAME: _list[SHIPPER][NAME] if SHIPPER in _list and NAME in _list[SHIPPER] else "", + SHIPPER+ADDRESS: _list[SHIPPER][ADDRESS] if SHIPPER in _list and ADDRESS in _list[SHIPPER] else "", + SHIPPER+POSTAL_CODE: _list[SHIPPER][POSTAL_CODE] if SHIPPER in _list and POSTAL_CODE in _list[SHIPPER] else "", + SHIPPER+PHONE_NUMBER: _list[SHIPPER][PHONE_NUMBER] if SHIPPER in _list and PHONE_NUMBER in _list[SHIPPER] else "", + NO_SIGNATURE_REQUIRED: _list[NO_SIGNATURE_REQUIRED] if NO_SIGNATURE_REQUIRED in _list else False, + TAILGATE_AUTHORIZED: _list[TAILGATE_AUTHORIZED] if TAILGATE_AUTHORIZED in _list else False } diff --git a/extraction/ocr.py b/servers/extraction/ocr.py similarity index 100% rename from extraction/ocr.py rename to servers/extraction/ocr.py diff --git a/extraction/requirements.txt b/servers/extraction/requirements.txt similarity index 100% rename from extraction/requirements.txt rename to servers/extraction/requirements.txt diff --git a/servers/tenant/kill-cluster.sh b/servers/kill-cluster.sh old mode 100644 new mode 100755 similarity index 100% rename from servers/tenant/kill-cluster.sh rename to servers/kill-cluster.sh diff --git a/servers/tenant/start-cluster.sh b/servers/start-cluster.sh similarity index 100% rename from servers/tenant/start-cluster.sh rename to servers/start-cluster.sh diff --git a/servers/tenant/Pipfile b/servers/tenant/Pipfile deleted file mode 100644 index db9acab..0000000 --- a/servers/tenant/Pipfile +++ /dev/null @@ -1,22 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -psycopg2-binary = "*" -celery = "==5.2.3" -flask-restplus = "==0.13.0" -flask-marshmallow = "==0.14.0" -marshmallow-sqlalchemy = "==0.24.1" -marshmallow = "==3.9.1" -python-dotenv = "==0.20.0" -SQLAlchemy = "*" -Flask = "==2.0.2" -Faker = "==13.7.0" -Flask-SQLAlchemy = "==2.4.4" - -[dev-packages] - -[requires] -python_version = "3.8" diff --git a/servers/tenant/Pipfile.lock b/servers/tenant/Pipfile.lock deleted file mode 100644 index a24e0f8..0000000 --- a/servers/tenant/Pipfile.lock +++ /dev/null @@ -1,501 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "6dff868c3d4497e3f717b753c5792b94a2f7cc606ce84194f3bb2bfd0fe58121" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "amqp": { - "hashes": [ - "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2", - "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359" - ], - "markers": "python_version >= '3.6'", - "version": "==5.1.1" - }, - "aniso8601": { - "hashes": [ - "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f", - "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973" - ], - "version": "==9.0.1" - }, - "attrs": { - "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" - }, - "billiard": { - "hashes": [ - "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", - "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" - ], - "version": "==3.6.4.0" - }, - "celery": { - "hashes": [ - "sha256:8aacd02fc23a02760686d63dde1eb0daa9f594e735e73ea8fb15c2ff15cb608c", - "sha256:e2cd41667ad97d4f6a2f4672d1c6a6ebada194c619253058b5f23704aaadaa82" - ], - "index": "pypi", - "version": "==5.2.3" - }, - "click": { - "hashes": [ - "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", - "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" - ], - "markers": "python_version >= '3.7'", - "version": "==8.1.3" - }, - "click-didyoumean": { - "hashes": [ - "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", - "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" - ], - "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", - "version": "==0.3.0" - }, - "click-plugins": { - "hashes": [ - "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", - "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" - ], - "version": "==1.1.1" - }, - "click-repl": { - "hashes": [ - "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", - "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" - ], - "version": "==0.2.0" - }, - "faker": { - "hashes": [ - "sha256:0301ace8365d98f3d0bf6e9a40200c8548e845d3812402ae1daf589effe3fb01", - "sha256:b1903db92175d78051858128ada397c7dc76f376f6967975419da232b3ebd429" - ], - "index": "pypi", - "version": "==13.7.0" - }, - "flask": { - "hashes": [ - "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2", - "sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a" - ], - "index": "pypi", - "version": "==2.0.2" - }, - "flask-marshmallow": { - "hashes": [ - "sha256:2adcd782b5a4a6c5ae3c96701f320d8ca6997995a52b2661093c56cc3ed24754", - "sha256:bd01a6372cbe50e36f205cfff0fc5dab0b7b662c4c8b2c4fc06a3151b2950950" - ], - "index": "pypi", - "version": "==0.14.0" - }, - "flask-restplus": { - "hashes": [ - "sha256:a15d251923a8feb09a5d805c2f4d188555910a42c64d58f7dd281b8cac095f1b", - "sha256:a66e442d0bca08f389fc3d07b4d808fc89961285d12fb8013f7cf15516fa9f5c" - ], - "index": "pypi", - "version": "==0.13.0" - }, - "flask-sqlalchemy": { - "hashes": [ - "sha256:05b31d2034dd3f2a685cbbae4cfc4ed906b2a733cff7964ada450fd5e462b84e", - "sha256:bfc7150eaf809b1c283879302f04c42791136060c6eeb12c0c6674fb1291fae5" - ], - "index": "pypi", - "version": "==2.4.4" - }, - "greenlet": { - "hashes": [ - "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3", - "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711", - "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd", - "sha256:049fe7579230e44daef03a259faa24511d10ebfa44f69411d99e6a184fe68073", - "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708", - "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67", - "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23", - "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1", - "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08", - "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd", - "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2", - "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa", - "sha256:356b3576ad078c89a6107caa9c50cc14e98e3a6c4874a37c3e0273e4baf33de8", - "sha256:40b951f601af999a8bf2ce8c71e8aaa4e8c6f78ff8afae7b808aae2dc50d4c40", - "sha256:572e1787d1460da79590bf44304abbc0a2da944ea64ec549188fa84d89bba7ab", - "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6", - "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc", - "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b", - "sha256:7418b6bfc7fe3331541b84bb2141c9baf1ec7132a7ecd9f375912eca810e714e", - "sha256:7cbd7574ce8e138bda9df4efc6bf2ab8572c9aff640d8ecfece1b006b68da963", - "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3", - "sha256:833e1551925ed51e6b44c800e71e77dacd7e49181fdc9ac9a0bf3714d515785d", - "sha256:8639cadfda96737427330a094476d4c7a56ac03de7265622fcf4cfe57c8ae18d", - "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe", - "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28", - "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3", - "sha256:903bbd302a2378f984aef528f76d4c9b1748f318fe1294961c072bdc7f2ffa3e", - "sha256:93f81b134a165cc17123626ab8da2e30c0455441d4ab5576eed73a64c025b25c", - "sha256:95e69877983ea39b7303570fa6760f81a3eec23d0e3ab2021b7144b94d06202d", - "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0", - "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497", - "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee", - "sha256:aa5b467f15e78b82257319aebc78dd2915e4c1436c3c0d1ad6f53e47ba6e2713", - "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58", - "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a", - "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06", - "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88", - "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965", - "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f", - "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4", - "sha256:be5f425ff1f5f4b3c1e33ad64ab994eed12fc284a6ea71c5243fd564502ecbe5", - "sha256:dd0b1e9e891f69e7675ba5c92e28b90eaa045f6ab134ffe70b52e948aa175b3c", - "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a", - "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1", - "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43", - "sha256:eb6ea6da4c787111adf40f697b4e58732ee0942b5d3bd8f435277643329ba627", - "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b", - "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168", - "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d", - "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5", - "sha256:f3acda1924472472ddd60c29e5b9db0cec629fbe3c5c5accb74d6d6d14773478", - "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf", - "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce", - "sha256:fa877ca7f6b48054f847b61d6fa7bed5cebb663ebc55e018fda12db09dcc664c", - "sha256:fdcec0b8399108577ec290f55551d926d9a1fa6cad45882093a7a07ac5ec147b" - ], - "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))", - "version": "==1.1.2" - }, - "importlib-resources": { - "hashes": [ - "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3", - "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8" - ], - "markers": "python_version < '3.9'", - "version": "==5.7.1" - }, - "itsdangerous": { - "hashes": [ - "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", - "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.2" - }, - "jinja2": { - "hashes": [ - "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", - "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" - ], - "markers": "python_version >= '3.7'", - "version": "==3.1.2" - }, - "jsonschema": { - "hashes": [ - "sha256:71b5e39324422543546572954ce71c67728922c104902cb7ce252e522235b33f", - "sha256:7c6d882619340c3347a1bf7315e147e6d3dae439033ae6383d6acb908c101dfc" - ], - "markers": "python_version >= '3.7'", - "version": "==4.5.1" - }, - "kombu": { - "hashes": [ - "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", - "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4" - ], - "markers": "python_version >= '3.7'", - "version": "==5.2.4" - }, - "markupsafe": { - "hashes": [ - "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", - "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88", - "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5", - "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7", - "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a", - "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603", - "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1", - "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135", - "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247", - "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6", - "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601", - "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77", - "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02", - "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e", - "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63", - "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f", - "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980", - "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b", - "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812", - "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff", - "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96", - "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1", - "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925", - "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a", - "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6", - "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e", - "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f", - "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4", - "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f", - "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3", - "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c", - "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a", - "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417", - "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a", - "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a", - "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37", - "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452", - "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933", - "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", - "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.1" - }, - "marshmallow": { - "hashes": [ - "sha256:73facc37462dfc0b27f571bdaffbef7709e19f7a616beb3802ea425b07843f4e", - "sha256:e26763201474b588d144dae9a32bdd945cd26a06c943bc746a6882e850475378" - ], - "index": "pypi", - "version": "==3.9.1" - }, - "marshmallow-sqlalchemy": { - "hashes": [ - "sha256:93f47b880ac7070f7b34c8ac0a71eeec3f8582a22e5c0330c1c436e3f5f99a37", - "sha256:d051cf013c075c43e1ee5c4b01f8fab6dd6b140dab6825be45875f674a0d289c" - ], - "index": "pypi", - "version": "==0.24.1" - }, - "prompt-toolkit": { - "hashes": [ - "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752", - "sha256:bd640f60e8cecd74f0dc249713d433ace2ddc62b65ee07f96d358e0b152b6ea7" - ], - "markers": "python_full_version >= '3.6.2'", - "version": "==3.0.29" - }, - "psycopg2-binary": { - "hashes": [ - "sha256:01310cf4cf26db9aea5158c217caa92d291f0500051a6469ac52166e1a16f5b7", - "sha256:083a55275f09a62b8ca4902dd11f4b33075b743cf0d360419e2051a8a5d5ff76", - "sha256:090f3348c0ab2cceb6dfbe6bf721ef61262ddf518cd6cc6ecc7d334996d64efa", - "sha256:0a29729145aaaf1ad8bafe663131890e2111f13416b60e460dae0a96af5905c9", - "sha256:0c9d5450c566c80c396b7402895c4369a410cab5a82707b11aee1e624da7d004", - "sha256:10bb90fb4d523a2aa67773d4ff2b833ec00857f5912bafcfd5f5414e45280fb1", - "sha256:12b11322ea00ad8db8c46f18b7dfc47ae215e4df55b46c67a94b4effbaec7094", - "sha256:152f09f57417b831418304c7f30d727dc83a12761627bb826951692cc6491e57", - "sha256:15803fa813ea05bef089fa78835118b5434204f3a17cb9f1e5dbfd0b9deea5af", - "sha256:15c4e4cfa45f5a60599d9cec5f46cd7b1b29d86a6390ec23e8eebaae84e64554", - "sha256:183a517a3a63503f70f808b58bfbf962f23d73b6dccddae5aa56152ef2bcb232", - "sha256:1f14c8b0942714eb3c74e1e71700cbbcb415acbc311c730370e70c578a44a25c", - "sha256:1f6b813106a3abdf7b03640d36e24669234120c72e91d5cbaeb87c5f7c36c65b", - "sha256:280b0bb5cbfe8039205c7981cceb006156a675362a00fe29b16fbc264e242834", - "sha256:2d872e3c9d5d075a2e104540965a1cf898b52274a5923936e5bfddb58c59c7c2", - "sha256:2f9ffd643bc7349eeb664eba8864d9e01f057880f510e4681ba40a6532f93c71", - "sha256:3303f8807f342641851578ee7ed1f3efc9802d00a6f83c101d21c608cb864460", - "sha256:35168209c9d51b145e459e05c31a9eaeffa9a6b0fd61689b48e07464ffd1a83e", - "sha256:3a79d622f5206d695d7824cbf609a4f5b88ea6d6dab5f7c147fc6d333a8787e4", - "sha256:404224e5fef3b193f892abdbf8961ce20e0b6642886cfe1fe1923f41aaa75c9d", - "sha256:46f0e0a6b5fa5851bbd9ab1bc805eef362d3a230fbdfbc209f4a236d0a7a990d", - "sha256:47133f3f872faf28c1e87d4357220e809dfd3fa7c64295a4a148bcd1e6e34ec9", - "sha256:526ea0378246d9b080148f2d6681229f4b5964543c170dd10bf4faaab6e0d27f", - "sha256:53293533fcbb94c202b7c800a12c873cfe24599656b341f56e71dd2b557be063", - "sha256:539b28661b71da7c0e428692438efbcd048ca21ea81af618d845e06ebfd29478", - "sha256:57804fc02ca3ce0dbfbef35c4b3a4a774da66d66ea20f4bda601294ad2ea6092", - "sha256:63638d875be8c2784cfc952c9ac34e2b50e43f9f0a0660b65e2a87d656b3116c", - "sha256:6472a178e291b59e7f16ab49ec8b4f3bdada0a879c68d3817ff0963e722a82ce", - "sha256:68641a34023d306be959101b345732360fc2ea4938982309b786f7be1b43a4a1", - "sha256:6e82d38390a03da28c7985b394ec3f56873174e2c88130e6966cb1c946508e65", - "sha256:761df5313dc15da1502b21453642d7599d26be88bff659382f8f9747c7ebea4e", - "sha256:7af0dd86ddb2f8af5da57a976d27cd2cd15510518d582b478fbb2292428710b4", - "sha256:7b1e9b80afca7b7a386ef087db614faebbf8839b7f4db5eb107d0f1a53225029", - "sha256:874a52ecab70af13e899f7847b3e074eeb16ebac5615665db33bce8a1009cf33", - "sha256:887dd9aac71765ac0d0bac1d0d4b4f2c99d5f5c1382d8b770404f0f3d0ce8a39", - "sha256:8b344adbb9a862de0c635f4f0425b7958bf5a4b927c8594e6e8d261775796d53", - "sha256:8fc53f9af09426a61db9ba357865c77f26076d48669f2e1bb24d85a22fb52307", - "sha256:91920527dea30175cc02a1099f331aa8c1ba39bf8b7762b7b56cbf54bc5cce42", - "sha256:93cd1967a18aa0edd4b95b1dfd554cf15af657cb606280996d393dadc88c3c35", - "sha256:99485cab9ba0fa9b84f1f9e1fef106f44a46ef6afdeec8885e0b88d0772b49e8", - "sha256:9d29409b625a143649d03d0fd7b57e4b92e0ecad9726ba682244b73be91d2fdb", - "sha256:a29b3ca4ec9defec6d42bf5feb36bb5817ba3c0230dd83b4edf4bf02684cd0ae", - "sha256:a9e1f75f96ea388fbcef36c70640c4efbe4650658f3d6a2967b4cc70e907352e", - "sha256:accfe7e982411da3178ec690baaceaad3c278652998b2c45828aaac66cd8285f", - "sha256:adf20d9a67e0b6393eac162eb81fb10bc9130a80540f4df7e7355c2dd4af9fba", - "sha256:af9813db73395fb1fc211bac696faea4ca9ef53f32dc0cfa27e4e7cf766dcf24", - "sha256:b1c8068513f5b158cf7e29c43a77eb34b407db29aca749d3eb9293ee0d3103ca", - "sha256:bda845b664bb6c91446ca9609fc69f7db6c334ec5e4adc87571c34e4f47b7ddb", - "sha256:c381bda330ddf2fccbafab789d83ebc6c53db126e4383e73794c74eedce855ef", - "sha256:c3ae8e75eb7160851e59adc77b3a19a976e50622e44fd4fd47b8b18208189d42", - "sha256:d1c1b569ecafe3a69380a94e6ae09a4789bbb23666f3d3a08d06bbd2451f5ef1", - "sha256:def68d7c21984b0f8218e8a15d514f714d96904265164f75f8d3a70f9c295667", - "sha256:dffc08ca91c9ac09008870c9eb77b00a46b3378719584059c034b8945e26b272", - "sha256:e3699852e22aa68c10de06524a3721ade969abf382da95884e6a10ff798f9281", - "sha256:e847774f8ffd5b398a75bc1c18fbb56564cda3d629fe68fd81971fece2d3c67e", - "sha256:ffb7a888a047696e7f8240d649b43fb3644f14f0ee229077e7f6b9f9081635bd" - ], - "index": "pypi", - "version": "==2.9.3" - }, - "pyrsistent": { - "hashes": [ - "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", - "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc", - "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e", - "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26", - "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec", - "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286", - "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045", - "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec", - "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8", - "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c", - "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca", - "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22", - "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a", - "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96", - "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc", - "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1", - "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07", - "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6", - "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b", - "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5", - "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6" - ], - "markers": "python_version >= '3.7'", - "version": "==0.18.1" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" - }, - "python-dotenv": { - "hashes": [ - "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f", - "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938" - ], - "index": "pypi", - "version": "==0.20.0" - }, - "pytz": { - "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" - ], - "version": "==2022.1" - }, - "setuptools": { - "hashes": [ - "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373", - "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e" - ], - "markers": "python_version >= '3.6'", - "version": "==59.6.0" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.16.0" - }, - "sqlalchemy": { - "hashes": [ - "sha256:09c606d8238feae2f360b8742ffbe67741937eb0a05b57f536948d198a3def96", - "sha256:166a3887ec355f7d2f12738f7fa25dc8ac541867147a255f790f2f41f614cb44", - "sha256:16abf35af37a3d5af92725fc9ec507dd9e9183d261c2069b6606d60981ed1c6e", - "sha256:2e885548da361aa3f8a9433db4cfb335b2107e533bf314359ae3952821d84b3e", - "sha256:2ec89bf98cc6a0f5d1e28e3ad28e9be6f3b4bdbd521a4053c7ae8d5e1289a8a1", - "sha256:2ecac4db8c1aa4a269f5829df7e706639a24b780d2ac46b3e485cbbd27ec0028", - "sha256:316c7e5304dda3e3ad711569ac5d02698bbc71299b168ac56a7076b86259f7ea", - "sha256:5041474dcab7973baa91ec1f3112049a9dd4652898d6a95a6a895ff5c58beb6b", - "sha256:53d2d9ee93970c969bc4e3c78b1277d7129554642f6ffea039c282c7dc4577bc", - "sha256:5864a83bd345871ad9699ce466388f836db7572003d67d9392a71998092210e3", - "sha256:5c90ef955d429966d84326d772eb34333178737ebb669845f1d529eb00c75e72", - "sha256:5d50cb71c1dbed70646d521a0975fb0f92b7c3f84c61fa59e07be23a1aaeecfc", - "sha256:64678ac321d64a45901ef2e24725ec5e783f1f4a588305e196431447e7ace243", - "sha256:64d796e9af522162f7f2bf7a3c5531a0a550764c426782797bbeed809d0646c5", - "sha256:6cb4c4f57a20710cea277edf720d249d514e587f796b75785ad2c25e1c0fed26", - "sha256:6e1fe00ee85c768807f2a139b83469c1e52a9ffd58a6eb51aa7aeb524325ab18", - "sha256:6e859fa96605027bd50d8e966db1c4e1b03e7b3267abbc4b89ae658c99393c58", - "sha256:7a052bd9f53004f8993c624c452dfad8ec600f572dd0ed0445fbe64b22f5570e", - "sha256:81e53bd383c2c33de9d578bfcc243f559bd3801a0e57f2bcc9a943c790662e0c", - "sha256:83cf3077712be9f65c9aaa0b5bc47bc1a44789fd45053e2e3ecd59ff17c63fe9", - "sha256:8b20c4178ead9bc398be479428568ff31b6c296eb22e75776273781a6551973f", - "sha256:8d07fe2de0325d06e7e73281e9a9b5e259fbd7cbfbe398a0433cbb0082ad8fa7", - "sha256:a0ae3aa2e86a4613f2d4c49eb7da23da536e6ce80b2bfd60bbb2f55fc02b0b32", - "sha256:af2587ae11400157753115612d6c6ad255143efba791406ad8a0cbcccf2edcb3", - "sha256:b3db741beaa983d4cbf9087558620e7787106319f7e63a066990a70657dd6b35", - "sha256:be094460930087e50fd08297db9d7aadaed8408ad896baf758e9190c335632da", - "sha256:cb441ca461bf97d00877b607f132772644b623518b39ced54da433215adce691", - "sha256:ce20f5da141f8af26c123ebaa1b7771835ca6c161225ce728962a79054f528c3", - "sha256:d57ac32f8dc731fddeb6f5d1358b4ca5456e72594e664769f0a9163f13df2a31", - "sha256:dce3468bf1fc12374a1a732c9efd146ce034f91bb0482b602a9311cb6166a920", - "sha256:e12532c4d3f614678623da5d852f038ace1f01869b89f003ed6fe8c793f0c6a3", - "sha256:e74ce103b81c375c3853b436297952ef8d7863d801dcffb6728d01544e5191b5", - "sha256:f0394a3acfb8925db178f7728adb38c027ed7e303665b225906bfa8099dc1ce8", - "sha256:f522214f6749bc073262529c056f7dfd660f3b5ec4180c5354d985eb7219801e", - "sha256:fbf8c09fe9728168f8cc1b40c239eab10baf9c422c18be7f53213d70434dea43", - "sha256:fca8322e04b2dde722fcb0558682740eebd3bd239bea7a0d0febbc190e99dc15" - ], - "index": "pypi", - "version": "==1.4.36" - }, - "vine": { - "hashes": [ - "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", - "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" - ], - "markers": "python_version >= '3.6'", - "version": "==5.0.0" - }, - "wcwidth": { - "hashes": [ - "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", - "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" - ], - "version": "==0.2.5" - }, - "werkzeug": { - "hashes": [ - "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", - "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.2" - }, - "zipp": { - "hashes": [ - "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", - "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" - ], - "markers": "python_version < '3.10'", - "version": "==3.8.0" - } - }, - "develop": {} -} diff --git a/servers/tenant/app.Dockerfile b/servers/tenant/app.Dockerfile deleted file mode 100644 index 0427bd6..0000000 --- a/servers/tenant/app.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.9 -EXPOSE 5000 -RUN apt-get update && apt-get -y install qpdf poppler-utils && apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev -COPY requirements.txt . -RUN pip3 install --upgrade pip -RUN pip3 install -r requirements.txt -WORKDIR /opt/metadata-extraction -ENV PYTHONPATH . \ No newline at end of file diff --git a/servers/tenant/blueprints/event_driven/pieces.py b/servers/tenant/blueprints/event_driven/pieces.py deleted file mode 100644 index 32bd159..0000000 --- a/servers/tenant/blueprints/event_driven/pieces.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -import datetime -from flask import request, jsonify, Blueprint - -import sys - -sys.path.insert(0, "..") # import parent folder - -from controllers.controllerMapper import PieceController -from models.models import TicketEvents, PieceEvents -from utils import ( - AlchemyEncoder, - alchemyConverter, -) -from flask_cognito_lib.decorators import auth_required - -pieces_bp = Blueprint("pieces_bp", __name__, url_prefix="piece") - -pieces_controller = PieceController() - - -""" -Route expects requests of format: - -{ - "piece_id" : "value", - "filters" : { - "field1": "value1", - "field2": "value2", - .... - } -} - -""" - - -@pieces_bp.route("/{piece_id}", methods=["GET"]) -@auth_required() -def pieces_get_history(piece_id): - filters = request.args.get("filters") - filters.extend({"piece_id": piece_id}) - - pieces = pieces_controller._get_latest_event_objects( - page=1, number_of_res=20, filters=filters - ) - - res = alchemyConverter(pieces) - response = json.dumps(res, cls=AlchemyEncoder) - - return response diff --git a/servers/tenant/blueprints/event_driven/ticket.py b/servers/tenant/blueprints/event_driven/ticket.py index f919bcd..450671a 100644 --- a/servers/tenant/blueprints/event_driven/ticket.py +++ b/servers/tenant/blueprints/event_driven/ticket.py @@ -2,7 +2,6 @@ from datetime import datetime from wsgiref import validate -from numpy import number from flask import make_response, request, jsonify, Blueprint import sys @@ -55,32 +54,46 @@ """ +# @ticket_bp.route("/status/", methods=["GET"]) +# # @auth_required() +# def ticket_get_all_with_status(status): # create ticket + +# limit = 5000 if "limit" not in request.args else request.args["limit"] +# sql_filters = get_clean_filters_dict(request.args) +# sql_filters["currentStatus"] = status +# data = ticket_status_controller._get(sql_filters, limit=limit) +# num_tickets = ticket_status_controller._get_count(sql_filters) + +# data = alchemyConverter(data) +# ticketIds = [x["ticketId"] for x in data] +# tickets = [] +# for ticketId in ticketIds: +# ticket = get_single(ticketId) +# if ticket: +# tickets.append(ticket) +# tickets = alchemyConverter(data) + +# res = {"tickets": tickets, "count": num_tickets} +# print(res) +# return make_response(json.dumps(res, cls=AlchemyEncoder)) @ticket_bp.route("/status/", methods=["GET"]) -@auth_required() +# @auth_required() def ticket_get_all_with_status(status): # create ticket limit = 5000 if "limit" not in request.args else request.args["limit"] - sql_filters = get_clean_filters_dict(request.args) - sql_filters["currentStatus"] = status - data = ticket_status_controller._get(sql_filters, limit=limit) - num_tickets = ticket_status_controller._get_count(sql_filters) - - data = alchemyConverter(data) - ticketIds = [x["ticketId"] for x in data] - tickets = [] - for ticketId in ticketIds: - ticket = get_single(ticketId) - if ticket: - tickets.append(ticket) - tickets = alchemyConverter(data) - - res = {"tickets": tickets, "count": num_tickets} + ticket_sql_filters = get_clean_filters_dict(request.args) + tickets = ticket_status_controller._get_tickets_with_status(status, ticket_sql_filters, limit) + num_tickets = ticket_status_controller._get_count(ticket_sql_filters) - return make_response(json.dumps(res, cls=AlchemyEncoder)) + res = {"tickets": alchemyConverter(tickets), "count": num_tickets} + return make_response(json.dumps(res, cls=AlchemyEncoder)) +""" +127.0.0.1:6767:/api/ticket/ +""" @ticket_bp.route("/", methods=["POST"]) -@auth_required() +# @auth_required() def ticket_post(): # create ticket print("Creating ticket from the following JSON:") print(request.data) @@ -98,7 +111,7 @@ def ticket_post(): # create ticket # TODO fix primary key issue, ticketeventID needs to be unique for edits @ticket_bp.route("/", methods=["POST"]) -@auth_required() +# @auth_required() def ticket_edit(ticket_id): # create ticket print("Creating ticket from the following JSON:") print(request.data) @@ -128,6 +141,11 @@ def get_clean_filters_dict(immutable_args): del sql_filters["limit"] return sql_filters +def corsify(resp): + resp = make_response(json.dumps(resp)) + resp.headers['Access-Control-Allow-Origin'] = '*' + resp.headers['Access-Control-Allow-Headers'] = ['Origin', 'X-Requested-With', 'Content-Type', 'Accept'] + return resp def validate_date_format(date_text): try: @@ -140,6 +158,9 @@ def default_start(): dt_start = validate_date_format("1900-01-01T00:00:00") return dt_start +def default_end(): + dt_end = validate_date_format("2100-01-01T00:00:00") + return dt_end def default_end(): dt_end = validate_date_format("2100-01-01T00:00:00") @@ -147,7 +168,7 @@ def default_end(): @ticket_bp.route("/", methods=["GET"]) -@auth_required() +# @auth_required() def ticket_get_all(): filters = request.args or {} sql_filters = get_clean_filters_dict(filters) @@ -163,7 +184,7 @@ def ticket_get_all(): data = ticket_controller._get_latest_event_objects_in_range( dt_start, dt_end, sql_filters, number_of_res=limit ) - + print(data) res = alchemyConverter(data) for ticket in res: ticket["pieces"] = ticket["pieces"].split(PIECES_SEPERATOR) @@ -190,10 +211,9 @@ def ticket_get(ticket_id): res = alchemyConverter(data) return make_response(json.dumps(res, cls=AlchemyEncoder)) +# return response """ -Route expects requests of format: - { "datetime" : "value", "filters" : { diff --git a/servers/tenant/blueprints/simple/document.py b/servers/tenant/blueprints/simple/document.py new file mode 100644 index 0000000..5836b86 --- /dev/null +++ b/servers/tenant/blueprints/simple/document.py @@ -0,0 +1,73 @@ +import os +from flask import request, jsonify, Blueprint +import io +from uuid import uuid4 +import traceback + +from celery_client import client, logger, fan_out, do_all_work +from tenant.models.models import DocumentStatus, Documents +from tenant.controllers.DocumentController import DocumentController, DocumentStatusController +import PyPDF2 +import extraction.app as ex +from celery import group +import json +from flask_cognito_lib.decorators import auth_required +from utils import ( + AlchemyEncoder, + alchemyConverter, +) + +document_bp = Blueprint("document_bp", __name__, url_prefix="document") + +FAILURE = -1 +SUCCESS = 0 +UPLOAD_FOLDER = "/opt/metadata-extraction/uploads" +document_status_controller = DocumentStatusController() +document_controller = DocumentController() + +@document_bp.route("/", methods=["POST"]) +# @auth_required() +def document_post(): + if "file" not in request.files: + res = jsonify({"message": "No file part in the request"}) + res.status_code = 400 + return res + + file = request.files["file"] + + if file.filename == "": + res = jsonify({"message": "No file selected for uploading"}) + res.status_code = 400 + return res + if file and file.filename.split(".")[-1].lower() == "pdf": + document_status = document_status_controller._create({"numPages": 0}) + response = {"documentStatusId": document_status.documentStatusId} + resp = jsonify(response) + resp.status_code = 202 + tasks_to_run = fan_out(file, document_status.documentStatusId) # split up tasks + do_all_work(tasks_to_run) # run ocr pipeline for each task + document_status = document_status_controller._modify({"documentStatusId": document_status.documentStatusId}, {"numPages": len(tasks_to_run)}) + return resp + else: + resp = jsonify({"message": "Allowed file types are pdf only"}) + resp.status_code = 400 + return resp + + +@document_bp.route("/", methods=["GET"]) +# @auth_required() +def document_get(document_id): + filters = {"documentStatusId": document_id} + documents = document_controller._get(filters) + ds_entry = document_status_controller._get(filters) + ds_entry = alchemyConverter(ds_entry) + num_pages = ds_entry[0]["numPages"] + documents = alchemyConverter(documents) + if len(documents) == num_pages: + res = {"status": "COMPLETE", "progress": 100, "documents": documents} + else: + res = {"status": "PENDING", "progress": 100*len(documents) // num_pages, "documents": []} + res = jsonify(res) + res.status_code = 200 + return res + diff --git a/servers/tenant/blueprints/simple/milestones.py b/servers/tenant/blueprints/simple/milestones.py index 8dad9f1..64e57f4 100644 --- a/servers/tenant/blueprints/simple/milestones.py +++ b/servers/tenant/blueprints/simple/milestones.py @@ -47,7 +47,7 @@ @milestone_bp.route("/", methods=["GET"]) -@auth_required() +# @auth_required() def milestone_get(ticket_id): # create ticket filters = { @@ -70,7 +70,7 @@ def milestone_get(ticket_id): # create ticket @milestone_bp.route("/", methods=["POST"]) -@auth_required() +# @auth_required() def milestone_post(milestone_type): # create ticket milestone_class = getattr(sys.modules[__name__], milestone_type) milestone_controller = class_to_cntrl_map[milestone_class] diff --git a/servers/tenant/celery.Dockerfile b/servers/tenant/celery.Dockerfile deleted file mode 100644 index d394f79..0000000 --- a/servers/tenant/celery.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.9 -RUN apt-get update && apt-get -y install qpdf poppler-utils && apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev -RUN apt -y install tesseract-ocr && apt -y install libtesseract-dev -COPY requirements.txt . -RUN pip3 install --upgrade pip -RUN pip3 install -r requirements.txt -WORKDIR /opt/metadata-extraction/server -ENV PYTHONPATH .. \ No newline at end of file diff --git a/servers/tenant/celery_client.py b/servers/tenant/celery_client.py index 7b7bedc..de16300 100644 --- a/servers/tenant/celery_client.py +++ b/servers/tenant/celery_client.py @@ -1,6 +1,105 @@ from celery import Celery from celery.utils.log import get_logger +import os +import io +from uuid import uuid4 +import traceback +import PyPDF2 +import extraction.app as ex +import extraction.extract as ext +from celery import group +from tenant.controllers.DocumentController import DocumentController +import boto3 +from botocore.client import Config + +# def get_file_s3(): +# s3_client = boto3.client('s3') +# TENANT = "test-tenant1" +# BUCKET = f"{TENANT}-bucket" +# OBJECT = 'signatures/cook-with-roommates-bonus-carbonara.jpg' + +# download_url = s3_client.generate_presigned_url( +# 'get_object', +# Params={'Bucket': BUCKET, 'Key': OBJECT, 'ResponseContentDisposition': 'attachment'}, +# ExpiresIn=3600) + +# view_url = s3_client.generate_presigned_url( +# 'get_object', +# Params={'Bucket': BUCKET, 'Key': OBJECT}, +# ExpiresIn=3600) + + +TENANT = "test-tenant1" +BUCKET = f"{TENANT}-bucket" +aws_access_key_id = os.getenv("aws_access_key_id") +aws_secret_access_key = os.getenv("aws_secret_access_key") +print(aws_secret_access_key, aws_access_key_id) CELERY_BROKER_URL = 'redis://redis:6379/0' client = Celery(__name__, broker=CELERY_BROKER_URL) logger = get_logger(__name__) +FAILURE = -1 +SUCCESS = 0 +PIECES_SEPERATOR = ",+-" +UPLOAD_FOLDER = "/opt/metadata-extraction/uploads" +s3 = boto3.resource('s3', region_name='ca-central-1', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, config=Config(signature_version='s3v4')) +bucket = s3.Bucket(BUCKET) +s3_client = boto3.client('s3', region_name='ca-central-1', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, config=Config(signature_version='s3v4')) + +def fan_out(file, documentStatusId): + folder_uuid = uuid4() + with io.BytesIO(file.read()) as open_pdf_file: + read_pdf = PyPDF2.PdfFileReader(open_pdf_file) + num_pages = read_pdf.getNumPages() + folder = f"{UPLOAD_FOLDER}/{folder_uuid}" + + os.mkdir(folder) + for i in range(num_pages): + output_pdf = PyPDF2.PdfFileWriter() + output_pdf.addPage(read_pdf.getPage(i)) + file_uuid = uuid4() + f_dir = f"{folder}/{file_uuid}" + os.mkdir(f_dir) + with open(f"{f_dir}/{file_uuid}.pdf", "wb") as f: + output_pdf.write(f) + + bucket.upload_file(f"{f_dir}/{file_uuid}.pdf", f"documents/{folder_uuid}/{file_uuid}.pdf") + file.close() + pdf_folders = os.listdir(folder) + return group([work.s(f"{folder}/{pdf_folder}", documentStatusId) for pdf_folder in pdf_folders]) + + +def do_all_work(tasks_to_run): + result = tasks_to_run.apply_async() + return result + + +@client.task +def work(pdf_folder, documentStatusId): + document_controller = DocumentController() + pdf_file = f"{pdf_folder}.pdf" + OBJECT = f"documents{pdf_file.replace(UPLOAD_FOLDER, '')}" + view_url = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': BUCKET, 'Key': OBJECT}, + ExpiresIn=3600) + try: + doclist = ex.work(pdf_folder) + doclist["orderS3Path"] = f"s3://{BUCKET}/{OBJECT}" + doclist["orderS3Link"] = view_url + doclist["pieces"] = PIECES_SEPERATOR.join(doclist["pieces"]) + doclist["documentStatusId"] = documentStatusId + doclist["success"] = True + document_controller._create(doclist) + except Exception as e: + logger.info(f"file {pdf_folder}/{pdf_file} error. msg: {str(e)}") + logger.info(traceback.format_exc()) + doclist = ext.generate_doclist({}) + doclist["orderS3Path"] = f"s3://{BUCKET}/{OBJECT}" + doclist["orderS3Link"] = view_url + doclist["pieces"] = PIECES_SEPERATOR.join(doclist["pieces"]) + doclist["documentStatusId"] = documentStatusId + doclist["success"] = False + document_controller._create(doclist) + return {"status": FAILURE, "folder": pdf_folder} + return {"status": SUCCESS, "folder": pdf_folder, "doclist": doclist} \ No newline at end of file diff --git a/servers/tenant/controllers/DocumentController.py b/servers/tenant/controllers/DocumentController.py new file mode 100644 index 0000000..9c09fdf --- /dev/null +++ b/servers/tenant/controllers/DocumentController.py @@ -0,0 +1,26 @@ +import os +from flask import request, jsonify, Blueprint + +from celery import group +import PyPDF2 +import io +from uuid import uuid4 +import sys + +from tenant.controllers.baseController import BaseController + +sys.path.insert(0, "..") # import parent folder + +from models.models import DocumentStatus, Documents +from models.__init__ import session +from utils import convert_dict_to_alchemy_filters + + +class DocumentController(BaseController): + def __init__(self): + super().__init__(Documents) + + +class DocumentStatusController(BaseController): + def __init__(self): + super().__init__(DocumentStatus) \ No newline at end of file diff --git a/servers/tenant/controllers/baseController.py b/servers/tenant/controllers/baseController.py index 8a95ce9..d7f7223 100644 --- a/servers/tenant/controllers/baseController.py +++ b/servers/tenant/controllers/baseController.py @@ -7,7 +7,6 @@ from sqlalchemy.orm import sessionmaker import sys from datetime import datetime - sys.path.insert(0, "..") # import parent folder from models.models import TicketStatus @@ -94,6 +93,7 @@ def _get(self, filters, limit=5000): return objects + def _get_count(self, filters): if not filters: filters = [] @@ -137,19 +137,18 @@ def _get_latest_event_objects(self, page=1, number_of_res=1, filters={}): # .all() # ) - print(*convert_dict_to_alchemy_filters(self.model, filters)) latest_objs = ( self.session.query(self.model) .distinct(self.model.non_prim_identifying_column_name) .filter(*convert_dict_to_alchemy_filters(self.model, filters)) - .order_by(self.model.timestamp) + .order_by(self.model.non_prim_identifying_column_name, self.model.timestamp) .limit(1) .all() ) # latest_objs = self.session.query(self.model, subquery).order_by(self.model.timestamp).all() - print("LATEST_OBJS-------") - print(latest_objs) + # print("LATEST_OBJS-------") + # print(latest_objs) return latest_objs # def _get_latest_event_objects_from_start_date(self, start_datetime, filters={}): @@ -178,7 +177,7 @@ def _get_latest_event_objects_in_range( self, datetime1, datetime2, filters={}, number_of_res=5 ): assert datetime1 <= datetime2 - time1 = int(time.mktime(datetime1.timetuple())) + time1 = max(0, int(time.mktime(datetime1.timetuple()))) time2 = int(time.mktime(datetime2.timetuple())) session_filters = convert_dict_to_alchemy_filters(self.model, filters) @@ -186,9 +185,6 @@ def _get_latest_event_objects_in_range( session_filters.append(self.model.timestamp >= time1) session_filters.append(self.model.timestamp <= time2) - print( - "------------------------RUNNING TICKET GET QUERY----------------------------" - ) results = ( self.session.query(self.model) .distinct(self.model.non_prim_identifying_column_name) @@ -197,9 +193,6 @@ def _get_latest_event_objects_in_range( .limit(number_of_res) .all() ) - print("----------complete-----------------") - for result in results: - print("TID " + str(result.ticketId)) return results def _find_latest_prim_key_from_non_prim_identifying_column_val( diff --git a/servers/tenant/controllers/controllerMapper.py b/servers/tenant/controllers/controllerMapper.py index 3797e09..2024fcc 100644 --- a/servers/tenant/controllers/controllerMapper.py +++ b/servers/tenant/controllers/controllerMapper.py @@ -1,12 +1,10 @@ from statistics import mode -from regex import D from controllers.baseController import ( BaseController, BaseTimeSeriesController, BaseNestedDependencyContoller, ) -import sys from models.models import * @@ -23,6 +21,33 @@ def __init__(self): class TicketStatusController(BaseController): def __init__(self): super().__init__(TicketStatus) + self.ticket_controller = BaseTimeSeriesController(TicketEvents) + + def _get_tickets_with_status(self, status, filters: dict, limit): + tickets = [] + + ticketIds = ( + self.session.query(TicketStatus.ticketId) + .filter(TicketStatus.currentStatus == status) + .limit(limit) + .all() + ) + + print(filters, ticketIds) + for i, tid_tup in enumerate(ticketIds): + + filters_cpy = filters.copy() + filters_cpy["ticketId"] = tid_tup[0] + print(filters_cpy) + + ticket = self.ticket_controller._get_latest_event_objects(filters=filters_cpy) + tickets.append(ticket[0]) + + if i == limit: + break + + + return tickets class MilestoneController(BaseController): @@ -220,4 +245,4 @@ def _create_base_event(self, args_dict): "approvedByUserId": args_dict["userId"], } ) - return obj \ No newline at end of file + return obj diff --git a/servers/tenant/models/models.py b/servers/tenant/models/models.py index 710dbf9..8832845 100644 --- a/servers/tenant/models/models.py +++ b/servers/tenant/models/models.py @@ -105,16 +105,26 @@ def __repr__(self): return f"< Users:: userId: {self.userId}>" +class DocumentStatus(Base): + __tablename__ = "documentstatus" + documentStatusId = Column(Integer, primary_key=True, nullable=False, autoincrement=True) + status = Column(String, nullable=False, default="PENDING") + numPages = Column(Integer, nullable=False) + + class Documents(Base): __tablename__ = "documents" documentId = Column(Integer, primary_key=True, nullable=False) + documentStatusId = Column(Integer, nullable=False) + success = Column(Boolean, nullable=False) timestamp = Column(Integer, default=int(time.time())) - barcodeNumber = Column(Integer, nullable=False) - houseReferenceNumber = Column(Integer, nullable=False) + barcodeNumber = Column(String, nullable=False) + houseReferenceNumber = Column(String, nullable=False) orderS3Link = Column(String, nullable=False) - weight = Column(Integer, nullable=False) + orderS3Path = Column(String, nullable=False) + weight = Column(String, nullable=False) claimedNumberOfPieces = Column(Integer, nullable=False) - BOLNumber = Column(Integer, nullable=False) + BOLNumber = Column(String, nullable=False) specialServices = Column(String) specialInstructions = Column(String) # shipper @@ -142,7 +152,6 @@ class TicketStatus(Base): currentStatus = Column(Enum(Generic_Milestone_Status), nullable=False) assignedTo = Column(Integer, ForeignKey(Users.userId), nullable=True, index=True) - user = relationship("Users") class TicketEvents(Base): @@ -156,12 +165,12 @@ class TicketEvents(Base): customerId = Column( Integer, ForeignKey(Customers.customerId), nullable=False, index=True ) - barcodeNumber = Column(Integer, nullable=False) - houseReferenceNumber = Column(Integer, nullable=False) + barcodeNumber = Column(String, nullable=False) + houseReferenceNumber = Column(String, nullable=False) orderS3Link = Column(String, nullable=False) - weight = Column(Integer, nullable=False) + weight = Column(String, nullable=False) claimedNumberOfPieces = Column(Integer, nullable=False) - BOLNumber = Column(Integer, nullable=False) + BOLNumber = Column(String, nullable=False) specialServices = Column(String) specialInstructions = Column(String) # shipper diff --git a/servers/tenant/requirements.txt b/servers/tenant/requirements.txt index 86417b1..8799b7a 100644 --- a/servers/tenant/requirements.txt +++ b/servers/tenant/requirements.txt @@ -1,3 +1,4 @@ +boto3 amqp==5.0.9 billiard==3.6.4.0 celery==5.2.3 diff --git a/servers/tenant/server.py b/servers/tenant/server.py index 87088d5..f4e56a9 100644 --- a/servers/tenant/server.py +++ b/servers/tenant/server.py @@ -6,6 +6,8 @@ from blueprints.simple.users import user_bp from blueprints.simple.milestones import milestone_bp from blueprints.simple.driver import driver_bp +from blueprints.simple.document import document_bp + from flask_cors import CORS from flask_cognito_lib import CognitoAuth @@ -32,6 +34,7 @@ parent.register_blueprint(user_bp) parent.register_blueprint(milestone_bp) parent.register_blueprint(driver_bp) +parent.register_blueprint(document_bp)