Merge branch 'develop' of github.com:aau-network-security/richkit into develop

Author: mrtrkmn

.dockerignore

@@ -4,3 +4,12 @@
 .github/logo
 .github/workflows
 
+
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+

Makefile

@@ -0,0 +1,49 @@
+.PHONY: all prep-dev venv clean lint test docker-test
+
+# virtual environment for development
+VENV_NAME?=venv
+VENV_ACTIVATE=. $(VENV_NAME)/bin/activate
+PYTHON=${VENV_NAME}/bin/python3
+# help messages for make, it runs in `make` or `make all`
+all:
+	@echo "\033[92m make prep-dev \033[0m"
+	@echo "---> Prepares dev environment, use only once"
+	@echo "\033[92m make test \033[0m"
+	@echo "---> Runs test cases in virtual environment"
+	@echo "\033[92m make lint \033[0m"
+	@echo "---> Linting project with autopep8"
+	@echo "\033[92m make clean \033[0m"
+	@echo "---> Cleans project cache and other stuffs"
+	@echo "\033[92m make docker-test \033[0m"
+	@echo "---> Runs test cases in docker environment"
+
+
+prep-dev:
+	python3 -m pip install virtualenv  ## virtual environment for development purposes
+	make venv
+
+venv: $(VENV_NAME)/bin/activate
+$(VENV_NAME)/bin/activate: requirements.txt
+		test -d $(VENV_NAME) || virtualenv -p python3 $(VENV_NAME)
+		${PYTHON} -m pip install -U pip setuptools
+		${PYTHON} -m pip install -U autopep8  coverage  isort
+		${PYTHON} -m pip install -U -r requirements.txt
+		touch $(VENV_NAME)/bin/activate
+
+clean:
+	rm -rf $(VENV_NAME) *.eggs *.egg-info dist build docs/_build .cache .coverage
+	rm -rf .pytest*  # cache file for Intellij PyCharm
+
+sort: venv
+	isort -rc . --skip_glob docs/*
+
+
+lint: venv
+	autopep8 --in-place --recursive --max-line-length=100 --exclude docs/source/conf.py,venv,__pycache__,old,build,dist .
+
+test: venv
+	coverage run --source=richkit -m pytest -Werror --ignore src/python-whois
+
+docker-test: clean
+	docker build -t richkit-docker-test -f Dockerfile.test .
+	docker run -e MAXMIND_LICENSE_KEY=$MAXMIND_LICENSE_KEY richkit-docker-test

richkit/analyse/analyse.py

@@ -123,11 +123,11 @@ def get_domain_name_features(domain):
     """
     domain_array = domain.split('.')
     num_tokens = len(domain_array)
-    len2ld  = len(get_sld(domain))
+    len2ld = len(get_sld(domain))
     len_domain = sum([len(el) for el in domain_array])
     domain_name_features = {
         "num_tokens": str(num_tokens),
-        "len2ld"    : str(len2ld),
+        "len2ld": str(len2ld),
         "len_domain": str(len_domain)
     }
     return domain_name_features
@@ -271,7 +271,6 @@ def ngram_count(domain, counts, counts_vc):
 
 
 def get_num_numeric_2ld(domain):
-
     """
 
     :param domain:
@@ -281,7 +280,6 @@ def get_num_numeric_2ld(domain):
 
 
 def get_radio_numeric_2ld(domain):
-
     """
 
     :param domain:

richkit/retrieve/cert_sh.py

@@ -40,7 +40,7 @@ def get_certificates(self, domain):
             return json.loads(content)
         except Exception as e:
             logger.error('Error while retrieving certificates: %s', e)
-            raise e
+            return None
 
     def get_all(self):
         """

richkit/test/analyse/test_analyse.py

@@ -153,7 +153,7 @@ def test_nld(self):
     def test_n_label(self):
         for k, v in self.domain.items():
             n_label3 = analyse.n_label(k, 3)
-            self.assertEqual(n_label3,v['n_label'])
+            self.assertEqual(n_label3, v['n_label'])
 
     def test_depth(self):
         for k, v in self.domain.items():

richkit/test/retrieve/test_urlvoid.py

@@ -36,7 +36,7 @@ def test_domain_registration_date(self):
         for k, v in self.test_urls.items():
             instance = URLVoid(k)
             domain_registration = instance.domain_registration_date()[:-15]
-            self.assertEqual(domain_registration,v["domain_registration"])
+            self.assertEqual(domain_registration, v["domain_registration"])
 
     def test_get_detection_rate(self):
         for k, v in self.test_urls.items():

richkit/test/retrieve/test_x509.py

@@ -5,7 +5,8 @@
 class Test_x509(unittest.TestCase):
 
     def setUp(self):
-        self.sans = ['*.google.com', 'mail.google.com', 'example.com', 'test.example.dk', 'test_domain.co.uk']
+        self.sans = ['*.google.com', 'mail.google.com',
+                     'example.com', 'test.example.dk', 'test_domain.co.uk']
 
     def test_unique_apex(self):
         assert unique_apex(self.sans) == 4
@@ -18,4 +19,3 @@ def test_unique_sld(self):
 
     def test_lcs(self):
         assert get_lcs_apex(self.sans) == 11
-