Draft: fixing search ranking

Signed-off-by: AbstractionFactory <[email protected]>

backend/internal/moduleindex/search.go

@@ -34,7 +34,8 @@ func (m moduleSearch) indexModuleVersion(ctx context.Context, addr ModuleAddr, m
 			"target_system": addr.TargetSystem,
 			"version":       string(response.ID),
 		},
-		ParentID: "",
+		ParentID:   "",
+		Popularity: module.Popularity,
 	}
 	if err := m.searchAPI.AddItem(ctx, versionItem); err != nil {
 		return err

backend/internal/providerindex/generator.go

@@ -281,7 +281,7 @@ func (d *documentationGenerator) scrapeProvider(ctx context.Context, addr provid
 			repoInfoFetched = true
 		}
 
-		providerVersion, err := d.scrapeVersion(ctx, addr, canonicalAddr, version, blocked, blockedReason)
+		providerVersion, err := d.scrapeVersion(ctx, addr, canonicalAddr, providerData, version, blocked, blockedReason)
 		if err != nil {
 			var repoNotFound *vcs.RepositoryNotFoundError
 			if errors.As(err, &repoNotFound) {
@@ -380,7 +380,7 @@ func (d *documentationGenerator) extractRepoInfo(ctx context.Context, addr provi
 	providerData.UpstreamForkCount = upstreamRepoInfo.ForkCount
 }
 
-func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr providertypes.ProviderAddr, canonicalAddr provider.Addr, version provider.Version, blocked bool, blockedReason string) (providertypes.ProviderVersion, error) {
+func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr providertypes.ProviderAddr, canonicalAddr provider.Addr, providerDetails *providertypes.Provider, version provider.Version, blocked bool, blockedReason string) (providertypes.ProviderVersion, error) {
 	// We get the VCS version before normalizing as the tag name may be different.
 	vcsVersion := version.Version.ToVCSVersion()
 	version.Version = version.Version.Normalize()
@@ -436,7 +436,7 @@ func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr provide
 		return versionData, fmt.Errorf("failed to store documentation for %s version %s (%w)", addr, version.Version, err)
 	}
 
-	if err := d.search.indexProviderVersion(ctx, addr.Addr, versionData); err != nil {
+	if err := d.search.indexProviderVersion(ctx, addr.Addr, providerDetails, versionData); err != nil {
 		return versionData, err
 	}
 

backend/internal/providerindex/search.go

@@ -17,8 +17,8 @@ type providerSearch struct {
 	searchAPI search.API
 }
 
-func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr provider.Addr, providerDetails providertypes.ProviderVersion) error {
-	version := providerDetails.ProviderVersionDescriptor.ID
+func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr provider.Addr, providerDetails *providertypes.Provider, providerVersionDetails providertypes.ProviderVersion) error {
+	version := providerVersionDetails.ProviderVersionDescriptor.ID
 	providerItem := searchtypes.IndexItem{
 		ID:          searchtypes.IndexID("providers/" + providerAddr.String()),
 		Type:        searchtypes.IndexTypeProvider,
@@ -31,7 +31,8 @@ func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr p
 			"name":      providerAddr.Name,
 			"version":   string(version),
 		},
-		ParentID: "",
+		ParentID:   "",
+		Popularity: providerDetails.Popularity,
 	}
 
 	if err := p.searchAPI.AddItem(ctx, providerItem); err != nil {
@@ -46,17 +47,17 @@ func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr p
 		{
 			"resource",
 			searchtypes.IndexTypeProviderResource,
-			providerDetails.Docs.Resources,
+			providerVersionDetails.Docs.Resources,
 		},
 		{
 			"datasource",
 			searchtypes.IndexTypeProviderDatasource,
-			providerDetails.Docs.DataSources,
+			providerVersionDetails.Docs.DataSources,
 		},
 		{
 			"function",
 			searchtypes.IndexTypeProviderFunction,
-			providerDetails.Docs.Functions,
+			providerVersionDetails.Docs.Functions,
 		},
 	} {
 		for _, docItem := range item.items {

backend/internal/search/searchtypes/index_item.go

@@ -17,11 +17,12 @@ type IndexItem struct {
 	LinkVariables map[string]string `json:"link"`
 	ParentID      IndexID           `json:"parent_id"`
 	LastUpdated   time.Time         `json:"last_updated"`
+	Popularity    int               `json:"popularity"`
 }
 
 func (i IndexItem) Equals(other IndexItem) bool {
 	if i.ID != other.ID || i.Type != other.Type || i.Addr != other.Addr || i.Version != other.Version ||
-		i.Title != other.Title || i.Description != other.Description || i.ParentID != other.ParentID {
+		i.Title != other.Title || i.Description != other.Description || i.ParentID != other.ParentID || i.Popularity != other.Popularity {
 		return false
 	}
 	if len(i.LinkVariables) != len(other.LinkVariables) {
@@ -60,5 +61,8 @@ func (i IndexItem) Validate() error {
 	if i.Version == "" {
 		return fmt.Errorf("the version field cannot be empty")
 	}
+	if i.Popularity < 0 {
+		return fmt.Errorf("the popularity cannot be negative")
+	}
 	return nil
 }

backend/internal/server/openapi.yml

@@ -1,8 +1,8 @@
 definitions:
     Addr:
         description: |-
-            Addr represents a full provider address (NAMESPACE/NAME). It currently translates to
-            github.com/NAMESPACE/terraform-provider-NAME .
+            Addr describes a module address combination of NAMESPACE-NAME-TARGETSYSTEM. This will translate to
+            github.com/NAMESPACE/terraform-TARGETSYSTEM-NAME for now.
         type: object
     BaseDetails:
         properties:
@@ -74,6 +74,9 @@ definitions:
                 type: object
             parent_id:
                 $ref: '#/definitions/IndexID'
+            popularity:
+                format: int64
+                type: integer
             title:
                 type: string
             type:

docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  postgres:
+    image: postgres
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: secret
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - type: bind
+        source: ./search/pg-indexer/schema.sql
+        target: /docker-entrypoint-initdb.d/schema.sql
+  frontend:
+    build:
+      context: frontend
+    volumes:
+      - source: ./frontend
+        target: /work
+        type: bind
+    ports:
+      - "127.0.0.1:8080:8080"
+  dataapi:
+    build:
+      context: search/worker
+    ports:
+      - "127.0.0.1:8787:8787"
+    volumes:
+      - source: ./search/worker
+        target: /work
+        type: bind
+    environment:
+      - DATABASE_URL=postgresql://posgres:secret@posgres/postgres

frontend/.gitignore

@@ -23,4 +23,6 @@ dist-ssr
 *.sln
 *.sw?
 
-.env
+.env
+
+.pnpm-store

frontend/Dockerfile

@@ -0,0 +1,8 @@
+FROM node:18
+
+WORKDIR /work
+RUN npm install -g pnpm
+VOLUME ["/work"]
+EXPOSE 3000
+
+CMD ["/bin/bash", "-c", "echo VITE_DATA_API_URL=http://dataapi:8787 >.env && pnpm i && pnpm run dev --host 0.0.0.0 --port 3000"]

search/pg-indexer/schema.sql

@@ -10,14 +10,20 @@ CREATE TABLE IF NOT EXISTS entities
     version        TEXT      NOT NULL,
     title          TEXT      NOT NULL,
     description    TEXT,
-    link_variables JSONB
+    link_variables JSONB,
+    document       TSVECTOR
 );
 
+CREATE INDEX IF NOT EXISTS idx_entities_title_lower
+    ON entities (lower(title));
+
 CREATE TABLE IF NOT EXISTS import_jobs
 (
     id           SERIAL PRIMARY KEY,
 
     created_at   TIMESTAMP NOT NULL,
     completed_at TIMESTAMP,
     successful   BOOLEAN
-);
+);
+
+ALTER TABLE entities ADD COLUMN popularity INT DEFAULT 0;

search/worker/Dockerfile

@@ -0,0 +1,7 @@
+FROM node:18
+
+WORKDIR /work
+VOLUME ["/work"]
+EXPOSE 8787
+
+CMD ["/bin/bash", "-c", "npm i && npm run dev"]