test: add testcases, support test report generation

Author: WangYihang

.gitignore

@@ -142,4 +142,5 @@ cython_debug/
 
 # Testing playground
 playground/
-test/repo/
+test/repo/
+test/report/

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+    - id: black
+      args: [
+        --line-length=121,
+      ]
+      exclude: ^(venv/|docs/)
+      types: ['python']
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+    - id: flake8
+      args: [
+        --max-line-length=121,
+        --ignore=E402
+      ]
+      exclude: ^(venv/|docs/)
+      types: ['python']

GitHacker/__init__.py

@@ -83,6 +83,44 @@ githacker --brute --url-file websites.txt --output-folder result
 - [x] Try common tags and branches when Index disabled
 - [x] [find packed refs](https://github.com/WangYihang/GitHacker/issues/1#issuecomment-487135667)
 
+## Test
+
+### Setup Development Environment
+
+```
+# Install docker and docker-compose
+apt install docker-desktop
+apt install docker-compose
+
+# Download GitHacker
+git clone https://github.com/WangYihang/GitHacker
+cd GitHacker
+python -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Run tests
+
+```
+# Generate testing repo
+python utils/gen.py
+
+# Run testcases
+sudo su
+source venv/bin/activate
+pip install -r requirements.txt
+python utils/test.py
+exit
+
+# Diff results
+python utils/diff.py
+```
+
+## Check report
+
+See `test/report/YYYY-MM-DD/index.html`
+
 ## Videos
 ### asciinema
 

README.md

@@ -1,6 +1,100 @@
-requests
-coloredlogs
-GitPython
-beautifulsoup4
-semver
-twine
+beautifulsoup4==4.11.1
+bleach==5.0.1
+certifi==2022.12.7
+cffi==1.15.1
+cfgv==3.3.1
+charset-normalizer==2.1.1
+coloredlogs==15.0.1
+commonmark==0.9.1
+cryptography==38.0.4
+distlib==0.3.6
+docutils==0.19
+filelock==3.8.2
+gitdb==4.0.10
+GitPython==3.1.29
+humanfriendly==10.0
+identify==2.5.9
+idna==3.4
+importlib-metadata==5.1.0
+jaraco.classes==3.2.3
+jeepney==0.8.0
+Jinja2==3.1.2
+keyring==23.11.0
+MarkupSafe==2.1.1
+more-itertools==9.0.0
+nodeenv==1.7.0
+pkginfo==1.9.2
+platformdirs==2.6.0
+pre-commit==2.20.0
+pycparser==2.21
+Pygments==2.13.0
+pytz==2022.6
+PyYAML==6.0
+readme-renderer==37.3
+requests==2.28.1
+requests-toolbelt==0.10.1
+rfc3986==2.0.0
+rich==12.6.0
+SecretStorage==3.3.3
+semver==2.13.0
+six==1.16.0
+smmap==5.0.0
+soupsieve==2.3.2.post1
+toml==0.10.2
+twine==4.0.2
+urllib3==1.26.13
+verboselogs==1.7
+virtualenv==20.17.1
+webencodings==0.5.1
+zipp==3.11.0
+zope.interface==5.5.2
+beautifulsoup4==4.11.1
+bleach==5.0.1
+certifi==2022.12.7
+cffi==1.15.1
+cfgv==3.3.1
+charset-normalizer==2.1.1
+coloredlogs==15.0.1
+commonmark==0.9.1
+cryptography==38.0.4
+distlib==0.3.6
+docutils==0.19
+filelock==3.8.2
+gitdb==4.0.10
+GitPython==3.1.29
+humanfriendly==10.0
+identify==2.5.9
+idna==3.4
+importlib-metadata==5.1.0
+jaraco.classes==3.2.3
+jeepney==0.8.0
+Jinja2==3.1.2
+keyring==23.11.0
+MarkupSafe==2.1.1
+more-itertools==9.0.0
+nodeenv==1.7.0
+pkginfo==1.9.2
+platformdirs==2.6.0
+pre-commit==2.20.0
+pycparser==2.21
+Pygments==2.13.0
+pytz==2022.6
+PyYAML==6.0
+readme-renderer==37.3
+requests==2.28.1
+requests-toolbelt==0.10.1
+rfc3986==2.0.0
+rich==12.6.0
+SecretStorage==3.3.3
+semver==2.13.0
+six==1.16.0
+smmap==5.0.0
+soupsieve==2.3.2.post1
+toml==0.10.2
+twine==4.0.2
+urllib3==1.26.13
+verboselogs==1.7
+virtualenv==20.17.1
+webencodings==0.5.1
+zipp==3.11.0
+zope.interface==5.5.2

requirements.txt

@@ -7,9 +7,9 @@
 setuptools.setup(
     name="GitHacker",
     version=GitHacker.__version__,
-    author="Wang Yihang",
+    author="Yihang Wang",
     author_email="[email protected]",
-    description="This is a multiple threads tool to download the `.git` folder and rebuild git repository locally.",
+    description="A multiple threads tool to download the `.git` folder and rebuild git repository locally.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/WangYihang/GitHacker",

setup.py

@@ -0,0 +1,29 @@
+<h2>Reports</h2>
+<style>
+table, th, td {
+  text-align: center;
+  border: 1px solid black;
+}
+</style>
+<table>
+  <thead>
+    <th>testcase</th>
+    <th>correct files</th>
+    <th>total files</th>
+    <th>correct ratio</th>
+    <th>different files</th>
+    <th>abscent files</th>
+  </thead>
+  <tbody>
+    {% for name, result in results.items() %}
+    <tr>
+      <td><a href="{{ name }}.html">{{ name }}</a></td>
+      <td>{{ result["correct"] }}</td>
+      <td>{{ result["total"] }}</td>
+      <td>{{ "%.4f"|format(result["ratio"]) }}%</td>
+      <td>{{ result["difference"]|length }}</td>
+      <td>{{ result["abscence"]|length }}</td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>

templates/index.html

@@ -0,0 +1,23 @@
+<title>{{ tool }}-v{{ version }}</title>
+<h2>Summary</h2>
+<div>
+  Date: {{ time }}<br>
+  Total files: {{ total }}, Correct files: {{ correct }}, Correct Ratio: {{ "%.4f"|format(ratio) }}%<br>
+  Different files: {{ difference|length }}, Absent files: {{ abscence|length }}<br>
+</div>
+
+<h2>Different file</h2>
+
+<table>
+  {% for filename in difference %}
+  <TR><TD>{{ filename }}</TD></TR>
+  {% endfor %}
+</table>
+  
+<h2>Absent file</h2>
+
+<table>
+  {% for filename in abscence %}
+  <TR><TD>{{ filename }}</TD></TR>
+  {% endfor %}
+</table>

templates/result.html

@@ -7,5 +7,5 @@ services:
     ports:
       - 80:80
     volumes:
-      - ./www:/usr/local/apache2/htdocs
+      - ../../repo:/usr/local/apache2/htdocs
       - ./httpd.conf:/usr/local/apache2/conf/httpd.conf

test/docker/apache-index-disabled/docker-compose.yml

@@ -7,4 +7,4 @@ services:
     ports:
       - 80:80
     volumes:
-      - ./www:/usr/local/apache2/htdocs
+      - ../../repo:/usr/local/apache2/htdocs

test/docker/apache-index-disabled/www

@@ -7,4 +7,4 @@ services:
     ports:
       - 80:80
     volumes:
-      - ./www:/usr/share/nginx/html
+      - ../../repo:/usr/share/nginx/html

test/docker/apache-index-enabled/docker-compose.yml

@@ -7,5 +7,5 @@ services:
     ports:
       - 80:80
     volumes:
-      - ./www:/usr/share/nginx/html
+      - ../../repo:/usr/share/nginx/html
       - ./nginx.conf:/etc/nginx/nginx.conf:ro

test/docker/apache-index-enabled/www

@@ -7,4 +7,4 @@ services:
     ports:
       - 80:80
     volumes:
-      - ./www:/var/www/html
+      - ../../repo:/var/www/html