diff --git a/backend/internal/moduleindex/search.go b/backend/internal/moduleindex/search.go index 1c8ffc8d..ad0b9e08 100644 --- a/backend/internal/moduleindex/search.go +++ b/backend/internal/moduleindex/search.go @@ -34,7 +34,8 @@ func (m moduleSearch) indexModuleVersion(ctx context.Context, addr ModuleAddr, m "target_system": addr.TargetSystem, "version": string(response.ID), }, - ParentID: "", + ParentID: "", + Popularity: module.Popularity, } if err := m.searchAPI.AddItem(ctx, versionItem); err != nil { return err diff --git a/backend/internal/providerindex/generator.go b/backend/internal/providerindex/generator.go index c40f5de8..8a08db70 100644 --- a/backend/internal/providerindex/generator.go +++ b/backend/internal/providerindex/generator.go @@ -281,7 +281,7 @@ func (d *documentationGenerator) scrapeProvider(ctx context.Context, addr provid repoInfoFetched = true } - providerVersion, err := d.scrapeVersion(ctx, addr, canonicalAddr, version, blocked, blockedReason) + providerVersion, err := d.scrapeVersion(ctx, addr, canonicalAddr, providerData, version, blocked, blockedReason) if err != nil { var repoNotFound *vcs.RepositoryNotFoundError if errors.As(err, &repoNotFound) { @@ -380,7 +380,7 @@ func (d *documentationGenerator) extractRepoInfo(ctx context.Context, addr provi providerData.UpstreamForkCount = upstreamRepoInfo.ForkCount } -func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr providertypes.ProviderAddr, canonicalAddr provider.Addr, version provider.Version, blocked bool, blockedReason string) (providertypes.ProviderVersion, error) { +func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr providertypes.ProviderAddr, canonicalAddr provider.Addr, providerDetails *providertypes.Provider, version provider.Version, blocked bool, blockedReason string) (providertypes.ProviderVersion, error) { // We get the VCS version before normalizing as the tag name may be different. vcsVersion := version.Version.ToVCSVersion() version.Version = version.Version.Normalize() @@ -436,7 +436,7 @@ func (d *documentationGenerator) scrapeVersion(ctx context.Context, addr provide return versionData, fmt.Errorf("failed to store documentation for %s version %s (%w)", addr, version.Version, err) } - if err := d.search.indexProviderVersion(ctx, addr.Addr, versionData); err != nil { + if err := d.search.indexProviderVersion(ctx, addr.Addr, providerDetails, versionData); err != nil { return versionData, err } diff --git a/backend/internal/providerindex/search.go b/backend/internal/providerindex/search.go index 3cf56c5c..d3c7e98e 100644 --- a/backend/internal/providerindex/search.go +++ b/backend/internal/providerindex/search.go @@ -17,8 +17,8 @@ type providerSearch struct { searchAPI search.API } -func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr provider.Addr, providerDetails providertypes.ProviderVersion) error { - version := providerDetails.ProviderVersionDescriptor.ID +func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr provider.Addr, providerDetails *providertypes.Provider, providerVersionDetails providertypes.ProviderVersion) error { + version := providerVersionDetails.ProviderVersionDescriptor.ID providerItem := searchtypes.IndexItem{ ID: searchtypes.IndexID("providers/" + providerAddr.String()), Type: searchtypes.IndexTypeProvider, @@ -31,7 +31,8 @@ func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr p "name": providerAddr.Name, "version": string(version), }, - ParentID: "", + ParentID: "", + Popularity: providerDetails.Popularity, } if err := p.searchAPI.AddItem(ctx, providerItem); err != nil { @@ -46,17 +47,17 @@ func (p providerSearch) indexProviderVersion(ctx context.Context, providerAddr p { "resource", searchtypes.IndexTypeProviderResource, - providerDetails.Docs.Resources, + providerVersionDetails.Docs.Resources, }, { "datasource", searchtypes.IndexTypeProviderDatasource, - providerDetails.Docs.DataSources, + providerVersionDetails.Docs.DataSources, }, { "function", searchtypes.IndexTypeProviderFunction, - providerDetails.Docs.Functions, + providerVersionDetails.Docs.Functions, }, } { for _, docItem := range item.items { diff --git a/backend/internal/search/searchtypes/index_item.go b/backend/internal/search/searchtypes/index_item.go index 8d5d64fd..f0739f94 100644 --- a/backend/internal/search/searchtypes/index_item.go +++ b/backend/internal/search/searchtypes/index_item.go @@ -17,11 +17,12 @@ type IndexItem struct { LinkVariables map[string]string `json:"link"` ParentID IndexID `json:"parent_id"` LastUpdated time.Time `json:"last_updated"` + Popularity int `json:"popularity"` } func (i IndexItem) Equals(other IndexItem) bool { if i.ID != other.ID || i.Type != other.Type || i.Addr != other.Addr || i.Version != other.Version || - i.Title != other.Title || i.Description != other.Description || i.ParentID != other.ParentID { + i.Title != other.Title || i.Description != other.Description || i.ParentID != other.ParentID || i.Popularity != other.Popularity { return false } if len(i.LinkVariables) != len(other.LinkVariables) { @@ -60,5 +61,8 @@ func (i IndexItem) Validate() error { if i.Version == "" { return fmt.Errorf("the version field cannot be empty") } + if i.Popularity < 0 { + return fmt.Errorf("the popularity cannot be negative") + } return nil } diff --git a/backend/internal/server/openapi.yml b/backend/internal/server/openapi.yml index 6c8f90c7..66539843 100644 --- a/backend/internal/server/openapi.yml +++ b/backend/internal/server/openapi.yml @@ -1,8 +1,8 @@ definitions: Addr: description: |- - Addr represents a full provider address (NAMESPACE/NAME). It currently translates to - github.com/NAMESPACE/terraform-provider-NAME . + Addr describes a module address combination of NAMESPACE-NAME-TARGETSYSTEM. This will translate to + github.com/NAMESPACE/terraform-TARGETSYSTEM-NAME for now. type: object BaseDetails: properties: @@ -74,6 +74,9 @@ definitions: type: object parent_id: $ref: '#/definitions/IndexID' + popularity: + format: int64 + type: integer title: type: string type: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3bbf56d6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +services: + postgres: + image: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: secret + ports: + - "127.0.0.1:5432:5432" + volumes: + - type: bind + source: ./search/pg-indexer/schema.sql + target: /docker-entrypoint-initdb.d/schema.sql + frontend: + build: + context: frontend + volumes: + - source: ./frontend + target: /work + type: bind + ports: + - "127.0.0.1:8080:8080" + dataapi: + build: + context: search/worker + ports: + - "127.0.0.1:8787:8787" + volumes: + - source: ./search/worker + target: /work + type: bind + environment: + - DATABASE_URL=postgresql://posgres:secret@posgres/postgres diff --git a/frontend/.gitignore b/frontend/.gitignore index 3b0b4037..f36b97bb 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -23,4 +23,6 @@ dist-ssr *.sln *.sw? -.env \ No newline at end of file +.env + +.pnpm-store \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..c59a19c6 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,8 @@ +FROM node:18 + +WORKDIR /work +RUN npm install -g pnpm +VOLUME ["/work"] +EXPOSE 3000 + +CMD ["/bin/bash", "-c", "echo VITE_DATA_API_URL=http://dataapi:8787 >.env && pnpm i && pnpm run dev --host 0.0.0.0 --port 3000"] diff --git a/search/pg-indexer/schema.sql b/search/pg-indexer/schema.sql index e670c03f..c24f2a15 100644 --- a/search/pg-indexer/schema.sql +++ b/search/pg-indexer/schema.sql @@ -10,9 +10,13 @@ CREATE TABLE IF NOT EXISTS entities version TEXT NOT NULL, title TEXT NOT NULL, description TEXT, - link_variables JSONB + link_variables JSONB, + document TSVECTOR ); +CREATE INDEX IF NOT EXISTS idx_entities_title_lower + ON entities (lower(title)); + CREATE TABLE IF NOT EXISTS import_jobs ( id SERIAL PRIMARY KEY, @@ -20,4 +24,6 @@ CREATE TABLE IF NOT EXISTS import_jobs created_at TIMESTAMP NOT NULL, completed_at TIMESTAMP, successful BOOLEAN -); \ No newline at end of file +); + +ALTER TABLE entities ADD COLUMN popularity INT DEFAULT 0; diff --git a/search/worker/Dockerfile b/search/worker/Dockerfile new file mode 100644 index 00000000..b59cd149 --- /dev/null +++ b/search/worker/Dockerfile @@ -0,0 +1,7 @@ +FROM node:18 + +WORKDIR /work +VOLUME ["/work"] +EXPOSE 8787 + +CMD ["/bin/bash", "-c", "npm i && npm run dev"]