Merge pull request #323 from JohT/feature/add-git-history-resports-using-treemaps

Add git history file overview treemap

Author: JohT

COMMANDS.md

@@ -264,7 +264,7 @@ Here is the resulting schema:
 
 #### Parameter
 
-The optional parameter `--source directory-path-to-the-source-folder-containing-git-repositories` can be used to select a different directory for the repositories. By default, the `source` directory within the analysis workspace directory is used. This command only needs the git history to be present. Therefore, `git clone --bare` is sufficient. If the `source` directory is also used for code analysis (like for Typescript) then a full git clone is of course needed.
+The optional parameter `--source directory-path-to-the-source-folder-containing-git-repositories` can be used to select a different directory for the repositories. By default, the `source` directory within the analysis workspace directory is used. This command only needs the git history to be present. Therefore, `git clone --bare` is sufficient. If the `source` directory is also used for code analysis (like for Typescript) then a full git clone is of course needed. Additionally, if you want to focus on a specific version or branch, use `--branch branch-name` to checkout the branch and `--single-branch` to exclude other branches before importing the git log data.
 
 #### Environment Variable
 

GETTING_STARTED.md

@@ -66,7 +66,7 @@ Use these optional command line options as needed:
 
 - If you want to analyze Typescript code, create a symbolic link inside the `source` directory that points to the Typescript project. Alternatively you can also copy the project into the `source` directory.
 
-- If you want to include git data like changed files and authors, create a symbolic link inside the `source` directory that points to the repository or clone it into the `source` directory. If you already have your Typescript project in there, you of course don't have to do it twice. If you are analyzing Java artifacts (full source not needed), it is sufficient to use a bare clone that only contains the git history without the sources using `git clone --bare`.
+- If you want to include git data like changed files and authors, create a symbolic link inside the `source` directory that points to the repository or clone it into the `source` directory. If you already have your Typescript project in there, you of course don't have to do it twice. If you are analyzing Java artifacts (full source not needed), it is sufficient to use a bare clone that only contains the git history without the sources using `git clone --bare`. If you want to focus on one branch, use `--branch branch-name` to checkout the branch and `--single-branch` to only fetch the history of that branch.
 
 - Alternatively to the steps above, run an already predefined download script
 

README.md

@@ -135,6 +135,7 @@ The [Code Structure Analysis Pipeline](./.github/workflows/internal-java-code-an
   - [numpy](https://numpy.org)
   - [pandas](https://pandas.pydata.org)
   - [pip](https://pip.pypa.io/en/stable)
+  - [plotly](https://plotly.com/python)
   - [monotonic](https://github.com/atdt/monotonic)
   - [Neo4j Python Driver](https://neo4j.com/docs/api/python-driver)
   - [openTSNE](https://github.com/pavlin-policar/openTSNE)

cypher/GitLog/List_git_files_per_commit_distribution.cypher

@@ -0,0 +1,6 @@
+// List how many git commits changed one file, how mandy changed two files, ....
+
+MATCH (git_commit:Git:Commit)-[:CONTAINS_CHANGE]->(git_change:Git:Change)-[]->(git_file:Git:File)
+ WITH git_commit, count(DISTINCT git_file.relativePath) AS filesPerCommit
+RETURN filesPerCommit, count(DISTINCT git_commit.sha)   AS commitCount
+ORDER BY filesPerCommit ASC

cypher/GitLog/List_git_files_with_commit_statistics_by_author.cypher

@@ -0,0 +1,24 @@
+// List git files with commit statistics
+
+ MATCH (git_file:File&Git&!Repository)
+ WHERE git_file.deletedAt IS NULL // filter out deleted files
+  WITH percentileDisc(git_file.createdAtEpoch, 0.5)          AS medianCreatedAtEpoch
+      ,percentileDisc(git_file.lastModificationAtEpoch, 0.5) AS medianLastModificationAtEpoch
+      ,collect(git_file)                                     AS git_files
+UNWIND git_files AS git_file
+  WITH *
+      ,datetime.fromepochMillis(coalesce(git_file.createdAtEpoch, medianCreatedAtEpoch))                                            AS fileCreatedAtTimestamp
+      ,datetime.fromepochMillis(coalesce(git_file.lastModificationAtEpoch, git_file.createdAtEpoch, medianLastModificationAtEpoch)) AS fileLastModificationAtTimestamp
+ MATCH (git_repository:Git&Repository)-[:HAS_FILE]->(git_file)
+ MATCH (git_commit:Git&Commit)-[:CONTAINS_CHANGE]->(git_change:Git&Change)-->(old_files_included:Git&File&!Repository)-[:HAS_NEW_NAME*0..3]->(git_file)
+RETURN git_repository.name + '/' + git_file.relativePath AS filePath
+      ,split(git_commit.author, ' <')[0]                 AS author
+      ,count(DISTINCT git_commit.sha)                    AS commitCount
+      ,date(max(git_commit.date))                        AS lastCommitDate
+      ,max(date(fileCreatedAtTimestamp))                 AS lastCreationDate
+      ,max(date(fileLastModificationAtTimestamp))        AS lastModificationDate
+      ,duration.inDays(date(max(git_commit.date)), date()).days               AS daysSinceLastCommit
+      ,duration.inDays(max(fileCreatedAtTimestamp), datetime()).days          AS daysSinceLastCreation
+      ,duration.inDays(max(fileLastModificationAtTimestamp), datetime()).days AS daysSinceLastModification
+      ,max(git_commit.sha)                               AS maxCommitSha
+ORDER BY filePath ASCENDING, commitCount DESCENDING

cypher/Validation/ValidateGitHistory.cypher

@@ -0,0 +1,6 @@
+// Check if there is at least one Git:Commit pointing to a Git:Change containing a Git:File from a Git:Repository
+
+ MATCH (commit:Git:Commit)-[:CONTAINS_CHANGE]->(change:Git:Change)-->(file:Git:File)
+ MATCH (repository:Git:Repository)-[:HAS_FILE]->(file)
+RETURN commit.sha AS commitSha
+ LIMIT 1

jupyter/ExternalDependenciesJava.ipynb

@@ -44,23 +44,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 235,
+   "execution_count": null,
    "id": "c1db254b",
    "metadata": {},
    "outputs": [],
    "source": [
     "def get_cypher_query_from_file(filename):\n",
     "    with open(filename) as file:\n",
-    "        return ' '.join(file.readlines())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 236,
-   "id": "59310f6f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "        return ' '.join(file.readlines())\n",
+    "\n",
+    "\n",
     "def query_cypher_to_data_frame(filename):\n",
     "    records, summary, keys = driver.execute_query(get_cypher_query_from_file(filename))\n",
     "    return pd.DataFrame([r.values() for r in records], columns=keys)"
@@ -1735,7 +1728,7 @@
   "celltoolbar": "Tags",
   "code_graph_analysis_pipeline_data_validation": "ValidateJavaExternalDependencies",
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "codegraph",
    "language": "python",
    "name": "python3"
   },
@@ -1749,7 +1742,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.9"
   },
   "title": "External Dependencies for Java"
  },

jupyter/ExternalDependenciesTypescript.ipynb

@@ -51,16 +51,9 @@
    "source": [
     "def get_cypher_query_from_file(filename):\n",
     "    with open(filename) as file:\n",
-    "        return ' '.join(file.readlines())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "59310f6f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "        return ' '.join(file.readlines())\n",
+    "\n",
+    "\n",
     "def query_cypher_to_data_frame(filename):\n",
     "    records, summary, keys = driver.execute_query(get_cypher_query_from_file(filename))\n",
     "    return pd.DataFrame([r.values() for r in records], columns=keys)"
@@ -1638,7 +1631,7 @@
   "celltoolbar": "Tags",
   "code_graph_analysis_pipeline_data_validation": "ValidateTypescriptModuleDependencies",
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "codegraph",
    "language": "python",
    "name": "python3"
   },
@@ -1652,7 +1645,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.9"
   },
   "title": "External Dependencies for Typescript"
  },