fix: github data (#13)

Author: yyyyaaa

.github/workflows/get-data.yml

@@ -1,10 +1,8 @@
 name: Fetch Contribution Data 🛠️
 
 on:
-  # push:
-  #   branches: [ main ]
-  # pull_request:
-  #   branches: [ main ]
+  push:
+    branches: [main]
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * 1" # Runs at 00:00 UTC on Monday
@@ -28,12 +26,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: "21"
+          node-version: "22"
 
       - name: Install dependencies
         run: yarn install
@@ -47,7 +45,7 @@ jobs:
           done
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_FE_DEPLOYER_ACCESS }}
           aws-secret-access-key: ${{ secrets.AWS_FE_DEPLOYER_SECRET }}
@@ -63,17 +61,28 @@ jobs:
 
       - name: Run it 🚀
         run: |
+          echo "Building project..."
           yarn build
+
+          echo "Setting up database schemas..."
           cd ./packages/stats-db
-          ./scripts/schema.sh
+          ./scripts/schema.sh -s npm
+          ./scripts/schema.sh -s github
 
+          echo "Fetching NPM data..."
           yarn npm:fetch:packages
           yarn npm:fetch:downloads
           yarn npm:report
           yarn npm:badges
           yarn npm:readme
+
+          echo "Fetching GitHub data..."
+          yarn gh:fetch
+
+          echo "Uploading database dump to S3..."
+          yarn db:dump:s3
         env:
-          # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DATABASE_URL: postgres://postgres:password@localhost:5432/example_db
 
           # AWS S3 Configuration
@@ -89,8 +98,8 @@ jobs:
         with:
           name: lib-count-badges
           path: |
-            badges/lib-count/total_downloads.json
-            badges/lib-count/monthly_downloads.json
-            badges/lib-count/utils_category.json
-            badges/lib-count/launchql_category.json
-            badges/lib-count/hyperweb_category.json
+            output/badges/lib-count/total_downloads.json
+            output/badges/lib-count/monthly_downloads.json
+            output/badges/lib-count/utils_category.json
+            output/badges/lib-count/launchql_category.json
+            output/badges/lib-count/hyperweb_category.json

packages/stats-db/README.md

@@ -106,3 +106,53 @@ To index from scratch, follow these steps in order:
    ```sh
    yarn npm:report && yarn npm:badges && yarn npm:readme
    ```
+
+# GitHub Analytics
+
+## **Project Overview**
+
+A TypeScript-based tool for collecting GitHub ecosystem data to map contributor networks and organizational relationships within the Cosmos blockchain ecosystem.
+
+## **Data Collection Requirements**
+
+### **1. Repository Collection**
+
+- **Target Organizations**: `hyperweb-io` and `launchql`
+- **Repository Filter**: Collect only non-fork repositories from each organization
+- **Repository Data**:
+  - Repository ID, name, and full name
+  - HTML URL and privacy status
+  - Fork status (to enable filtering)
+
+### **2. Contributor Collection**
+
+- **Scope**: All contributors to all non-fork repositories collected in step 1
+- **Contributor Data**:
+  - GitHub username (login)
+  - User ID
+  - Contribution count per repository
+  - Total contributions across all repositories
+
+### **3. Organization Network Discovery**
+
+- **Scope**: All public organizations that any contributor (from step 2) belongs to
+- **Organization Data**:
+  - Organization login/name
+  - Organization API URL
+  - Unique organization list (deduplicated across all contributors)
+
+### **Data Collection Flow**
+
+1. Fetch all repositories from `hyperweb-io` and `launchql` organizations
+2. Filter out forked repositories, keeping only original repositories
+3. For each non-fork repository, fetch complete contributor list
+4. For each unique contributor discovered, fetch their public organization memberships
+5. Aggregate and deduplicate all discovered organizations
+
+### **Output Requirements**
+
+- **Non-fork repositories**: Organized by parent organization
+- **Contributor profiles**: Including cross-repository contribution mapping
+- **Organization network**: Complete deduplicated list of all public organizations discovered through contributor analysis
+
+This data collection strategy enables comprehensive ecosystem analysis by mapping the full network of organizations connected through shared contributors in the target GitHub organizations.

packages/stats-db/package.json

@@ -30,9 +30,9 @@
     "npm:report": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node ./src/tasks/npm/npm.tasks.ts generate:report",
     "npm:badges": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node ./src/tasks/npm/npm.tasks.ts generate:badges",
     "npm:readme": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node ./src/tasks/npm/npm.tasks.ts generate:readme",
-    "gh:fetch": "ts-node src/tasks/github/github.tasks.ts fetch",
-    "gh:report": "ts-node src/tasks/github/github.tasks.ts report",
-    "gh:export": "ts-node src/tasks/github/github.tasks.ts export",
+    "gh:fetch": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node src/tasks/github/github.tasks.ts fetch",
+    "gh:report": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node src/tasks/github/github.tasks.ts report",
+    "gh:export": "DATABASE_URL=postgres://postgres:password@localhost:5432/example_db ts-node src/tasks/github/github.tasks.ts export",
     "gh:analyze": "ts-node src/tasks/github/analyze-repo.ts",
     "db:dump": "ts-node ./src/dump.ts --no-upload",
     "db:dump:s3": "ts-node ./src/dump.ts",
@@ -48,6 +48,8 @@
     "@interweb/fetch-api-client": "^0.6.0",
     "@interweb/http-errors": "^0.1.0",
     "@octokit/graphql": "^8.2.1",
+    "@octokit/plugin-retry": "^8.0.1",
+    "@octokit/plugin-throttling": "^11.0.1",
     "@octokit/rest": "^21.1.1",
     "@types/pg": "^8.11.10",
     "dotenv": "^16.4.7",

packages/stats-db/scripts/github.sql

@@ -28,10 +28,24 @@ CREATE TABLE github.author (
     login text NOT NULL,
     name text,
     avatar_url text,
+    primary_email text, -- Most frequently used email from commits
     created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
     updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
 );
 
+-- Create author emails table to track all emails used by contributors
+CREATE TABLE github.author_email (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+    author_id uuid NOT NULL REFERENCES github.author(id),
+    email text NOT NULL,
+    commit_count integer NOT NULL DEFAULT 1, -- How many commits used this email
+    first_seen_at timestamp with time zone NOT NULL,
+    last_seen_at timestamp with time zone NOT NULL,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (author_id, email)
+);
+
 -- Create the repositories table
 CREATE TABLE github.repository (
     id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
@@ -42,6 +56,10 @@ CREATE TABLE github.repository (
     url text NOT NULL,
     is_fork boolean NOT NULL DEFAULT false,
     fork_date timestamp with time zone,
+    parent_repo text, -- Full name of parent repository (e.g., "owner/repo")
+    source_repo text, -- Full name of ultimate source repository if different from parent
+    fork_detection_method text, -- 'github_api', 'known_forks', 'commit_analysis', 'name_similarity', 'manual_verification'
+    fork_detection_confidence text, -- 'high', 'medium', 'low'
     owner_id uuid NOT NULL REFERENCES github.organization(id),
     stars_count integer NOT NULL DEFAULT 0,
     forks_count integer NOT NULL DEFAULT 0,
@@ -104,9 +122,15 @@ CREATE TABLE github.contribution_summary (
 -- Create indexes for better query performance
 CREATE INDEX idx_repository_owner ON github.repository(owner_id);
 CREATE INDEX idx_repository_fork_date ON github.repository(fork_date) WHERE fork_date IS NOT NULL;
+CREATE INDEX idx_repository_parent_repo ON github.repository(parent_repo) WHERE parent_repo IS NOT NULL;
+CREATE INDEX idx_repository_source_repo ON github.repository(source_repo) WHERE source_repo IS NOT NULL;
+CREATE INDEX idx_repository_fork_detection ON github.repository(fork_detection_method, fork_detection_confidence) WHERE is_fork = true;
 CREATE INDEX idx_daily_contribution_repo_date ON github.daily_contribution(repository_id, date);
 CREATE INDEX idx_daily_contribution_author_date ON github.daily_contribution(author_id, date);
 CREATE INDEX idx_author_org_history_dates ON github.author_organization_history(author_id, organization_id, joined_at);
+CREATE INDEX idx_author_email_author ON github.author_email(author_id);
+CREATE INDEX idx_author_email_email ON github.author_email(email);
+CREATE INDEX idx_author_email_commit_count ON github.author_email(author_id, commit_count DESC);
 
 -- Add indexes for org connection queries
 CREATE INDEX idx_org_connection_source ON github.organization_connection(source_org_id);