update: update crawling and fix #19 and #24

Author: repoog

.gitignore

@@ -1,4 +1,4 @@
 *.bak
-
 *.pyc
-*.log
+*.log
+Config.py

GitPrey.py

@@ -13,6 +13,7 @@
     print("[!]Error: You have to install BeautifulSoup module.")
     exit()
 
+import os
 import re
 import math
 import sys
@@ -61,8 +62,7 @@ class GitPrey(object):
     def __init__(self, keyword):
         self.keyword = keyword
         self.search_url = "https://github.com/search?o=desc&p={page}&q={keyword}&ref=searchresults&s=indexed&type=Code&utf8=%E2%9C%93"
-        self.headers = {
-            'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"}
+        self.headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"}
         self.cookies = ""
 
     def search_project(self):
@@ -72,7 +72,7 @@ def search_project(self):
         """
         unique_project_list = []
         self.__auto_login(USER_NAME, PASSWORD)
-        info_print('[*] Searching projects hard...')
+        info_print('[*] Searching hard for projects...')
 
         # Get unique project list of first page searched results
         total_progress = SCAN_DEEP[SEARCH_LEVEL - 1]
@@ -105,16 +105,16 @@ def __page_project_list(page_html):
         :returns: Project list of per page
         """
         cur_par_html = BeautifulSoup(page_html, "lxml")
-        project_info = cur_par_html.select("a.text-bold")
-        page_project = [project.text for project in project_info]
+        project_info = cur_par_html.select("a.link-gray")
+        page_project = [project.text.strip() for project in project_info]
         return page_project
 
     def sensitive_info_query(self, project_string, mode):
         """
         Search sensitive information and sensitive file from projects
         :param project_string: Key words string for querying
         :param mode: Searching mode within "content" or "filename"
-        :returns: None
+        :returns: Code segments or file lists
         """
         if mode == "content":
             # Output code line with sensitive key words like username.
@@ -132,16 +132,17 @@ def sensitive_info_query(self, project_string, mode):
         if mode == "filename":
             # Search project according to file path.
             path_sig_list = self.__pattern_db_list(PATH_DB)
-            path_string = " filename:" + " filename:".join(path_sig_list) + project_string
+            path_string = "filename:" + " filename:".join(path_sig_list) + project_string
             repo_file_dic = self.__file_name_inspect(path_string, print_mode=1)
             return repo_file_dic
 
     def __file_content_inspect(self, project_string, file_pattern, project_pattern):
         """
         Check sensitive code in particular project
-        :param content_query_string: Content string for searching
-        :param info_sig_match: information signature match regular
-        :returns: None
+        :param project_string: Projects for searching
+        :param file_pattern: File string for searching
+        :param project_pattern: Content signature match regular
+        :returns: Code segments
         """
         query_string = " OR ".join(project_pattern)
         repo_file_dic = self.__file_name_inspect(query_string + project_string + file_pattern)
@@ -169,15 +170,16 @@ def __file_name_inspect(self, file_query_string, print_mode=0):
         """
         Inspect sensitive file in particular project
         :param file_query_string: File string for searching
-        :returns: None
+        :param print_mode: 1 means print file, 0 means print code
+        :returns: Files lists
         """
         page_num = 1
         repo_file_dic = {}
         while page_num <= SCAN_DEEP[SEARCH_LEVEL - 1]:
             check_url = self.search_url.format(page=page_num, keyword=file_query_string)
             page_html = self.__get_page_html(check_url)
             project_html = BeautifulSoup(page_html, 'lxml')
-            repo_list = project_html.select('div .min-width-0 > a:nth-of-type(2)')
+            repo_list = project_html.select('a[data-hydro-click-hmac]')
             if not repo_list:
                 break
             # Handle file links for each project
@@ -205,7 +207,7 @@ def __pattern_db_list(file_path):
         :returns: Signature item list
         """
         item_list = []
-        with open(file_path, 'r') as pattern_file:
+        with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as pattern_file:
             item_line = pattern_file.readline()
             while item_line:
                 item_list.append(item_line.strip())
@@ -239,7 +241,7 @@ def __auto_login(self, username, password):
         for item in input_items:
             post_data[item.get('name')] = item.get('value')
         post_data['login'], post_data['password'] = username, password
-        login_request.post("https://github.com/session", data=post_data, headers=self.headers)
+        login_request.post("https://github.com/session", data=post_data, cookies=login_html.cookies, headers=self.headers)
         self.cookies = login_request.cookies
         if self.cookies['logged_in'] == 'no':
             error_print('[!] Error: Login Github failed, please check account in config file.')
@@ -280,18 +282,16 @@ def is_keyword_valid(keyword):
 def init():
     """
     Initialize GitPrey with module inspection and input inspection
-    :return: None
+    :return: Key words
     """
     if not importlib.util.find_spec('lxml'):
-        error_print('[!]Error: You have to install lxml module.')
+        error_print('[!] Error: You have to install lxml module.')
         exit()
 
     # Get command parameters for searching level and key words
     parser = argparse.ArgumentParser(description="Searching sensitive file and content in GitHub.")
-    parser.add_argument("-l", "--level", type=int, choices=range(1, 6), default=1, metavar="level",
-                        help="Set search level within 1~5, default is 1.")
-    parser.add_argument("-k", "--keywords", metavar="keywords", required=True,
-                        help="Set key words to search projects.")
+    parser.add_argument("-l", "--level", type=int, choices=range(1, 6), default=1, metavar="level", help="Set search level within 1~5, default is 1.")
+    parser.add_argument("-k", "--keywords", metavar="keywords", required=True, help="Set key words to search projects.")
     args = parser.parse_args()
 
     SEARCH_LEVEL = args.level if args.level else 1
@@ -309,7 +309,6 @@ def init():
 
     return key_words
 
-
 def project_miner(key_words):
     """
     Search projects for content and path inspection later.
@@ -323,6 +322,9 @@ def project_miner(key_words):
     project_info_output = "\n[*] Found {num} public projects related to the key words.\n"
     info_print(project_info_output.format(num=len(total_project_list)))
 
+    if (len(total_project_list) == 0):
+        exit(0)
+
     # Join all projects to together to search
     repo_string = " repo:" + " repo:".join(total_project_list)
 

README.md

@@ -1,7 +1,7 @@
 ## GitHub敏感信息扫描工具
 ![](https://img.shields.io/github/license/repoog/GitPrey.svg)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/repoog/GitPrey.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Frepoog%2FGitPrey)
-### 功能设计说明
+### 功能设计
 GitPrey是根据企业关键词进行项目检索以及相应敏感文件和敏感文件内容扫描的工具，其设计思路如下：
 * 根据关键词在GitHub中进行全局代码内容和路径的搜索（in:file,path），将项目结果做项目信息去重整理得到所有关键词相关的项目，即疑似项目结果；
 * 基于PATTERN_DB中的敏感文件名或敏感代码对所有疑似项目做文件名搜索（filename:）和代码搜索（in:file）；
@@ -20,15 +20,15 @@ GitPrey是根据企业关键词进行项目检索以及相应敏感文件和敏
 
 深度选择与企业扫描周期性应该成正相关，深度选择小，则相应扫描的周期性也应当较小，如深度选择为Level 1，则相应的扫描周期基于企业情况可定为每天或每周，深度选择为Level 5，则相应的扫描周期可适当延长。例如，关键词“Google”最大（Level 5）可搜索两天前上传的项目代码，而关键词“repoog”搜索结果则不足1页。
 
-### 技术实现说明
+### 技术实现
 项目配置文件Config.py中需要配置使用者的Github用户名、密码：
 * 未登录Github进行代码搜索会因为请求速度过快（约10页代码结果页）而返回HTTP STATUE 429，即Too Many Requests的错误，因此需要登录后进行搜索；
 * 在项目内关键词文件名和关键词内容扫描时未采用API，原因有两点：一是搜索代码的API频率限制很大（认证后30次/分钟）无法满足快速搜索；二是某些项目关键词的搜索结果项超过100条，而API在设置per_page参数后至多支持展现100条结果项；
 项目配置文件Config.py中需要配置FILE_DB/INFO_DB/PASS_DB/PATH_DB项，用途如下：_
 * 敏感文件搜索是基于配置项中的PATH_DB内容检索特定文件的泄漏；
 * 敏感内容搜索是基于PASS_DB和FILE_DB进行检索，再根据INFO_DB和PASS_DB输出相关代码行；_
 
-### 程序使用帮助
+### 程序使用
 GitPrey v2.2版本后去除了ACCESS_TOKEN的配置以及配置文件中的SEARCH_LEVEL和KEYWORDS配置项，改用命令行参数方式执行：
 ```
 USAGE:
@@ -40,13 +40,13 @@ USAGE:
 * -k：必填参数，用于设置搜索关键词，若关键词中包含空白字符，需用双引号将关键词括起来；
 * -h：帮助信息。
 
-### 文件配置说明
+### 文件配置
 pattern为搜索项文件配置目录，相关文件说明如下：
 * path.db：敏感文件名或文件后缀，用于搜索文件名，如：htpasswd
 * file.db：敏感内容关键词搜索的文件名称范围，内容搜索在该文件名称范围内进行，如：.env
 * info.db：敏感内容关键词（由于AND/OR/NOT操作符在Github单次搜索中最多支持五个，故关键词会进行轮询），如：password
 
-### 程序更新列表
+### 程序更新
 * v1.0 初始版本
 * v2.0 更新搜索设计和算法
 * v2.1 更新搜索结果输出展现

config/Config.py

@@ -5,4 +5,4 @@
 
 # GitHub account config for searching
 USER_NAME = ""
-PASSWORD = ""
+PASSWORD = ""

include/ColorPrint.py

@@ -7,12 +7,13 @@
     exit()
 
 import logging
+import os
 
 init(autoreset=True)
 
 logger = logging.getLogger('')
 logger.setLevel(logging.INFO)
-file_handle = logging.FileHandler('GitPrey.log')
+file_handle = logging.FileHandler(os.path.join(os.path.dirname(__file__), '../GitPrey.log'))
 file_handle.setLevel(logging.INFO)
 formatter = logging.Formatter('%(message)s')
 file_handle.setFormatter(formatter)
@@ -28,7 +29,6 @@ def error_print(string):
 def info_print(string):
     # Print information with green color
     print(Fore.GREEN + string)
-    logger.info(string)
 
 
 def project_print(string):

pattern/info.db

@@ -12,4 +12,7 @@ jdbc
 password
 passwd
 pass
-pwd
+pwd
+public_key
+publickey
+private_ke