-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathget_emails.py
117 lines (99 loc) · 5.31 KB
/
get_emails.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
# -*- coding: utf-8 -*-
# Author : xxx
# Date :
# Description: 获取邮件列表
import base64
import poplib
import zipfile
import rarfile
import traceback
from io import BytesIO
from datetime import datetime
from transaction.resources.config import *
from transaction.utils.logger import logger
from transaction.get_files import get_files_from_zip, get_files_from_rar
from transaction.get_compressed_pkg import get_mail_list
from transaction.db.mysql_conn_pool import MySQLConnector
from transaction.db.sql_stm import origin_max_time_select_sql, origin_insert_sql
def parse(pop, conn, max_email_id, max_email_receive_time):
"""
邮件解析入口
:param pop: POP3客户端
:param conn: 数据库连接
:param max_email_id: 表中最大邮件索引
:param max_email_receive_time: 表中最大邮件接收时间
:return:
"""
for data, item in get_mail_list(pop, max_email_id, max_email_receive_time):
item['compressed_file_name'] = data.get("name", "")
try:
if data['name'].endswith(".zip"): # zip格式的压缩文件
zip_obj = zipfile.ZipFile(BytesIO(base64.b64decode(data['file_data'].encode())))
logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题："
+ item["email_subject"] + " ------------------ 解析开始 ")
origin_json_records = [] # 原始数据json
origin_json_value_records = [] # 原始数据值
for record in get_files_from_zip(zip_obj, item):
record["raw_data"] = str(record["raw_data"])
origin_json_records.append(record.copy())
origin_json_value_records.append(tuple(record.values()))
if len(origin_json_value_records) == 0:
continue
# 写入外部数据库
conn.execute_many(pool, origin_insert_sql, origin_json_value_records)
logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题：" + item[
"email_subject"] + " ------------------ 解析结束 \n")
# logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题：" + item["email_subject"] + " ------------------ 清洗开始 ")
# clean(origin_json_records) # 数据清洗
# logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题：" + item["email_subject"] + " ------------------ 清洗结束 \n")
zip_obj.close()
elif data['name'].endswith(".rar"): # rar格式的压缩文件
rar_obj = rarfile.RarFile(BytesIO(base64.b64decode(data['file_data'].encode())))
logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题："
+ item["email_subject"] + " ------------------ 解析开始 ")
origin_json_records = [] # 原始数据json
origin_json_value_records = [] # 原始数据值
for record in get_files_from_rar(rar_obj, item):
record["raw_data"] = str(record["raw_data"])
origin_json_records.append(record.copy())
origin_json_value_records.append(tuple(record.values()))
if len(origin_json_value_records) == 0:
continue
# 写入外部数据库
conn.execute_many(pool, origin_insert_sql, origin_json_value_records)
logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题："
+ item["email_subject"] + " ------------------ 解析结束 \n")
# logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题：" + item["email_subject"] + " ------------------ 清洗开始 ")
# clean(origin_json_records) # 数据清洗
# logger.info(str(datetime.now()) + " 邮件索引：" + str(item["email_id"]) + " --- 邮件主题：" + item["email_subject"] + " ------------------ 清洗结束 \n")
rar_obj.close()
except Exception as e:
logger.error(traceback.format_exc())
if __name__ == '__main__':
pop = poplib.POP3_SSL(mail_account.get('host', ""))
try:
# 邮箱登录
pop.user(mail_account.get('user', ""))
pop.pass_(mail_account.get("password", ""))
except poplib.error_proto as e:
logger.error("Login failed: " + e)
else:
conn = None
max_email_id = -1
max_email_receive_time = ""
try:
# 初始化MySQL连接池
conn = MySQLConnector()
pool = conn.create_pool()
# 获取表中最大邮件索引, 以从下一个索引开始获取邮件
result_set = conn.fetch_one(pool, origin_max_time_select_sql)
max_email_id = result_set[0]
max_email_receive_time = result_set[1]
except Exception as e:
logger.error(traceback.format_exc())
else:
# 数据抽取解析
parse(pop, conn, max_email_id, max_email_receive_time)
finally:
# 退出邮箱
pop.quit()