Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build(pre-commit): pre-commit autoupdate by ci #1211

Open
wants to merge 4 commits into
base: dependency-updates
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ repos:
exclude: \.dot$
- id: debug-statements
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.7.2
rev: v0.9.4
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- repo: https://github.com/pre-commit/mirrors-eslint
rev: v9.14.0
rev: v9.19.0
hooks:
- id: eslint
args: [--fix, --color]
Expand All @@ -36,7 +36,7 @@ repos:
- [email protected]
- [email protected]
- repo: https://github.com/crate-ci/typos
rev: v1.27.0
rev: typos-dict-v0.12.4
hooks:
- id: typos
exclude: |
Expand Down
2 changes: 1 addition & 1 deletion conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def fixtures():


@pytest.fixture(scope='session')
def django_db_setup(django_db_setup, django_db_blocker, fixtures): # noqa: PT004 - pytest-django requires this name "django_db_setup"
def django_db_setup(django_db_setup, django_db_blocker, fixtures):
"""Populate database with test data from fixtures directories."""
with django_db_blocker.unblock():
call_command('loaddata', *fixtures)
Expand Down
178 changes: 76 additions & 102 deletions rdmo/core/management/commands/find_spam_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,144 +9,118 @@
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'-t', '--timespan', default=2, type=int,
help='timespan in seconds between two joining users, ' +
'less than the given value is considered to be suspicious ' +
', default is 2'
"-t", "--timespan",
default=2, type=int,
help=(
"timespan in seconds between two joining users, "
"less than the given value is considered to be suspicious, "
"default is 2"
)
)
parser.add_argument(
'-n', '--occurrence', default=3, type=int,
help='number of sequentially occurring timespan ' +
'violations at which users are put into the ' +
'potential spam users list, default is 3'
"-n", "--occurrence",
default=3, type=int,
help=(
"number of sequentially occurring timespan "
"violations at which users are put into the "
"potential spam users list, default is 3"
)
)
parser.add_argument(
'-p', '--print', action='store_true',
help='print found users, don\'t save them to csv'
"-p", "--print",
action="store_true",
help="print found users, don't save them to csv"
)
parser.add_argument(
'-o', '--output_file', default='potential_spam_users.csv',
help='output file, default is \'potential_spam_users.csv\''
"-o", "--output_file",
default="potential_spam_users.csv",
help="output file, default is 'potential_spam_users.csv'"
)

def save_csv(self, data, filename):
if len(data) > 0:
data_file = open(filename, 'w')
csv_writer = csv.writer(data_file)
csv_writer.writerow(list(data[0].keys()))
for user in data:
csv_writer.writerow(user.values())
print('List written to ' + filename)
if data:
with open(filename, "w", newline="", encoding="utf-8") as data_file:
csv_writer = csv.writer(data_file)
csv_writer.writerow(list(data[0].keys()))
for user in data:
csv_writer.writerow(user.values())
print(f"List written to {filename}")

def print_file(self, filename):
f = open(filename)
content = f.read()
print(content)
f.close()
with open(filename) as f:
content = f.read()
print(content)

def get_users_having_projects(self):
arr = []
memberships = Membership.objects.all().values_list('user')
for mem in memberships:
arr.append(
User.objects.filter(id=mem[0]).values('id')[0]['id']
)
return arr
memberships = Membership.objects.all().values_list("user", flat=True)
return list(memberships)

def append_to_group(
self, group_list, group_count, user, list_users_having_projects
):
date_string = '%Y-%m-%dT%H:%M:%S.%f'
last_login = user['last_login']
if last_login is not None:
last_login = last_login.strftime(date_string)
def append_to_group(self, group_list, group_count, user, list_users_having_projects):
date_string = "%Y-%m-%dT%H:%M:%S.%f"
last_login = user["last_login"].strftime(date_string) if user["last_login"] else None

has_project = user['id'] in list_users_having_projects
has_project = user["id"] in list_users_having_projects
group_list[group_count].append(
{
'id': user['id'],
'email': user['email'],
'username': user['username'],
'first_name': user['first_name'],
'last_name': user['last_name'],
'date_joined': user['date_joined'].strftime(date_string),
'last_login': last_login,
'has_project': has_project,
"id": user["id"],
"email": user["email"],
"username": user["username"],
"first_name": user["first_name"],
"last_name": user["last_name"],
"date_joined": user["date_joined"].strftime(date_string),
"last_login": last_login,
"has_project": has_project,
}
)
return group_list

def find_potential_spam_users(self, timespan, occurrence):
list_users_having_projects = self.get_users_having_projects()
arr = []
for idx, user in enumerate(User.objects.all().order_by('date_joined')):
arr.append(
{
'id': user.id,
'username': user.username,
'first_name': user.first_name,
'last_name': user.last_name,
'date_joined': user.date_joined,
'unix_joined': user.date_joined.timestamp(),
'email': user.email,
'last_login': user.last_login,
}
)
arr = [
{
"id": user.id,
"username": user.username,
"first_name": user.first_name,
"last_name": user.last_name,
"date_joined": user.date_joined,
"unix_joined": user.date_joined.timestamp(),
"email": user.email,
"last_login": user.last_login,
}
for user in User.objects.all().order_by("date_joined")
]

grouped = {}
group_count = 0
for idx, user in enumerate(arr):
prev = None
diff = timespan
if idx > 0:
prev = arr[idx-1]
diff = user['unix_joined'] - prev['unix_joined']
prev = arr[idx - 1] if idx > 0 else None
diff = user["unix_joined"] - prev["unix_joined"] if prev else timespan

if prev is not None and diff >= timespan:
if prev and diff >= timespan:
group_count += 1

try:
grouped[group_count]
except KeyError:
grouped[group_count] = []
grouped.setdefault(group_count, [])
grouped = self.append_to_group(grouped, group_count, user, list_users_having_projects)

grouped = self.append_to_group(
grouped, group_count, user, list_users_having_projects
)

no_potential_spam_users = 0
grouped_clean = {}
for group_id in grouped:
group = grouped[group_id]
if len(group) > occurrence:
no_potential_spam_users += len(group)
grouped_clean[group_id] = group

potential_spam_users = []
for group in grouped_clean:
for user in grouped_clean[group]:
potential_spam_users.append(user)
grouped_clean = {k: v for k, v in grouped.items() if len(v) > occurrence}
potential_spam_users = [user for group in grouped_clean.values() for user in group]

return (potential_spam_users, len(list_users_having_projects))
return potential_spam_users, len(list_users_having_projects)

def handle(self, *args, **options):
no_total_users = User.objects.count()
print(f"Total no of users: {no_total_users}")

no_total_users = User.objects.all().count()
print('Total no of users: %d' % (no_total_users))
potential_spam_users, no_users_having_projects =\
self.find_potential_spam_users(
options['timespan'], options['occurrence']
)
potential_spam_users, no_users_having_projects = self.find_potential_spam_users(
options["timespan"], options["occurrence"]
)

percentage = (100 / no_total_users) * len(potential_spam_users) if no_total_users else 0
print(
'Potential spam users: %d %.2f%% / of which have at least one project %d'
% (
len(potential_spam_users),
(100/no_total_users)*len(potential_spam_users),
no_users_having_projects
)
f"Potential spam users: {len(potential_spam_users)} {percentage:.2f}% / "
f"of which have at least one project {no_users_having_projects}"
)

self.save_csv(potential_spam_users, options['output_file'])
if options['print'] is True:
self.print_file(options['output_file'])
self.save_csv(potential_spam_users, options["output_file"])
if options["print"]:
self.print_file(options["output_file"])
Loading
Loading