From e2701c3474207710310e7fe58a3e9229083df265 Mon Sep 17 00:00:00 2001 From: Anup Cowkur Date: Sat, 2 Nov 2024 19:39:42 +0530 Subject: [PATCH] feat(backend): use short codes for list filters closes #1408 --- backend/api/filter/appfilter.go | 78 ++++++----- backend/api/filter/shortfilters.go | 99 ++++++++++++++ backend/api/main.go | 1 + backend/api/measure/app.go | 61 +++++++++ backend/cleanup/cleanup/cleanup.go | 21 ++- frontend/dashboard/app/api/api_calls.ts | 122 ++++++++++-------- .../app/components/exceptions_details.tsx | 4 +- frontend/dashboard/app/utils/auth/auth.ts | 4 +- ...41102075104_create_short_filters_table.sql | 17 +++ 9 files changed, 314 insertions(+), 93 deletions(-) create mode 100644 backend/api/filter/shortfilters.go create mode 100644 self-host/postgres/20241102075104_create_short_filters_table.sql diff --git a/backend/api/filter/appfilter.go b/backend/api/filter/appfilter.go index 5d892ace0..85b6eecb2 100644 --- a/backend/api/filter/appfilter.go +++ b/backend/api/filter/appfilter.go @@ -3,8 +3,10 @@ package filter import ( "backend/api/pairs" "backend/api/server" - "backend/api/text" "context" + "crypto/md5" + "encoding/hex" + "encoding/json" "fmt" "slices" "time" @@ -50,6 +52,9 @@ type AppFilter struct { // client Timezone string `form:"timezone"` + // Represents a short code for list filters + FilterShortCode string `form:"filter_short_code"` + // Versions is the list of version string // to be matched & filtered on. Versions []string `form:"versions"` @@ -141,6 +146,20 @@ type FilterList struct { DeviceNames []string `json:"device_names"` } +// Hash generates an MD5 hash of the FilterList struct. +func (f *FilterList) Hash() (string, error) { + data, err := json.Marshal(f) + if err != nil { + return "", err + } + + // Compute MD5 hash + md5Hash := md5.Sum(data) + + // Convert hash to hex string + return hex.EncodeToString(md5Hash[:]), nil +} + // Versions represents a list of // (version, code) pairs. type Versions struct { @@ -227,48 +246,43 @@ func (af *AppFilter) OSVersionPairs() (osVersions *pairs.Pairs[string, string], // Expand expands comma separated fields to slice // of strings func (af *AppFilter) Expand() { - if len(af.Versions) > 0 { - af.Versions = text.SplitTrimEmpty(af.Versions[0], ",") + filters, err := GetFiltersFromFilterShortCode(af.FilterShortCode, af.AppID) + if err != nil { + return } - if len(af.VersionCodes) > 0 { - af.VersionCodes = text.SplitTrimEmpty(af.VersionCodes[0], ",") + if len(filters.Versions) > 0 { + af.Versions = filters.Versions } - - if len(af.OsNames) > 0 { - af.OsNames = text.SplitTrimEmpty(af.OsNames[0], ",") + if len(filters.VersionCodes) > 0 { + af.VersionCodes = filters.VersionCodes } - - if len(af.OsVersions) > 0 { - af.OsVersions = text.SplitTrimEmpty(af.OsVersions[0], ",") + if len(filters.OsNames) > 0 { + af.OsNames = filters.OsNames } - - if len(af.Countries) > 0 { - af.Countries = text.SplitTrimEmpty(af.Countries[0], ",") + if len(filters.OsVersions) > 0 { + af.OsVersions = filters.OsVersions } - - if len(af.DeviceNames) > 0 { - af.DeviceNames = text.SplitTrimEmpty(af.DeviceNames[0], ",") + if len(filters.Countries) > 0 { + af.Countries = filters.Countries } - - if len(af.DeviceManufacturers) > 0 { - af.DeviceManufacturers = text.SplitTrimEmpty(af.DeviceManufacturers[0], ",") + if len(filters.DeviceNames) > 0 { + af.DeviceNames = filters.DeviceNames } - - if len(af.Locales) > 0 { - af.Locales = text.SplitTrimEmpty(af.Locales[0], ",") + if len(filters.DeviceManufacturers) > 0 { + af.DeviceManufacturers = filters.DeviceManufacturers } - - if len(af.NetworkProviders) > 0 { - af.NetworkProviders = text.SplitTrimEmpty(af.NetworkProviders[0], ",") + if len(filters.DeviceLocales) > 0 { + af.Locales = filters.DeviceLocales } - - if len(af.NetworkTypes) > 0 { - af.NetworkTypes = text.SplitTrimEmpty(af.NetworkTypes[0], ",") + if len(filters.NetworkProviders) > 0 { + af.NetworkProviders = filters.NetworkProviders } - - if len(af.NetworkGenerations) > 0 { - af.NetworkGenerations = text.SplitTrimEmpty(af.NetworkGenerations[0], ",") + if len(filters.NetworkTypes) > 0 { + af.NetworkTypes = filters.NetworkTypes + } + if len(filters.NetworkGenerations) > 0 { + af.NetworkGenerations = filters.NetworkGenerations } } diff --git a/backend/api/filter/shortfilters.go b/backend/api/filter/shortfilters.go new file mode 100644 index 000000000..210b4954b --- /dev/null +++ b/backend/api/filter/shortfilters.go @@ -0,0 +1,99 @@ +package filter + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "backend/api/chrono" + "backend/api/server" + + "github.com/google/uuid" + "github.com/leporo/sqlf" +) + +type ShortFilters struct { + Code string + AppId uuid.UUID + Filters FilterList + UpdatedAt time.Time + CreatedAt time.Time +} + +type ShortFiltersPayload struct { + Filters FilterList `json:"filters"` +} + +func (shortFilters *ShortFilters) MarshalJSON() ([]byte, error) { + apiMap := make(map[string]any) + apiMap["code"] = shortFilters.Code + apiMap["app_id"] = shortFilters.AppId + apiMap["filters"] = shortFilters.Filters + apiMap["created_at"] = shortFilters.CreatedAt.Format(chrono.ISOFormatJS) + apiMap["updated_at"] = shortFilters.UpdatedAt.Format(chrono.ISOFormatJS) + return json.Marshal(apiMap) +} + +func NewShortFilters(appId uuid.UUID, filters FilterList) (*ShortFilters, error) { + hash, err := filters.Hash() + if err != nil { + return nil, err + } + + return &ShortFilters{ + Code: hash, + AppId: appId, + Filters: filters, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }, nil +} + +func (shortFilters *ShortFilters) Create() error { + // If already exists, just return + _, err := GetFiltersFromFilterShortCode(shortFilters.Code, shortFilters.AppId) + if err == nil { + return nil + } + + stmt := sqlf.PostgreSQL.InsertInto("public.short_filters"). + Set("code", shortFilters.Code). + Set("app_id", shortFilters.AppId). + Set("filters", shortFilters.Filters). + Set("created_at", shortFilters.CreatedAt). + Set("updated_at", shortFilters.UpdatedAt) + defer stmt.Close() + + _, err = server.Server.PgPool.Exec(context.Background(), stmt.String(), stmt.Args()...) + if err != nil { + return err + } + + return nil +} + +// Returns filters for a given short code and appId. If it doesn't exist, returns an error +func GetFiltersFromFilterShortCode(filterShortCode string, appId uuid.UUID) (*FilterList, error) { + var filters FilterList + + stmt := sqlf.PostgreSQL. + Select("filters"). + From("public.short_filters"). + Where("code = ?", filterShortCode). + Where("app_id = ?", appId) + defer stmt.Close() + + err := server.Server.PgPool.QueryRow(context.Background(), stmt.String(), stmt.Args()...).Scan(&filters) + + if err != nil { + fmt.Printf("Error fetching filters from filter short code %v: %v\n", filterShortCode, err) + return nil, err + } + + return &filters, nil +} + +func (shortFilters *ShortFilters) String() string { + return fmt.Sprintf("ShortFilters - code: %s, app_id: %s, filters: %v, created_at: %v, updated_at: %v ", shortFilters.Code, shortFilters.AppId, shortFilters.Filters, shortFilters.CreatedAt, shortFilters.UpdatedAt) +} diff --git a/backend/api/main.go b/backend/api/main.go index 3c014c4fc..a9011d54c 100644 --- a/backend/api/main.go +++ b/backend/api/main.go @@ -109,6 +109,7 @@ func main() { apps.GET(":id/settings", measure.GetAppSettings) apps.PATCH(":id/settings", measure.UpdateAppSettings) apps.PATCH(":id/rename", measure.RenameApp) + apps.POST(":id/shortFilters", measure.CreateShortFilters) } teams := r.Group("/teams", measure.ValidateAccessToken()) diff --git a/backend/api/measure/app.go b/backend/api/measure/app.go index 294cfb167..29b1a3e58 100644 --- a/backend/api/measure/app.go +++ b/backend/api/measure/app.go @@ -5066,3 +5066,64 @@ func RenameApp(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"ok": "done"}) } + +func CreateShortFilters(c *gin.Context) { + userId := c.GetString("userId") + appId, err := uuid.Parse(c.Param("id")) + if err != nil { + msg := `app id invalid or missing` + fmt.Println(msg, err) + c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + return + } + + app := App{ + ID: &appId, + } + + team, err := app.getTeam(c) + if err != nil { + msg := "failed to get team from app id" + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + return + } + if team == nil { + msg := fmt.Sprintf("no team exists for app [%s]", app.ID) + c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + return + } + + ok, err := PerformAuthz(userId, team.ID.String(), *ScopeAppRead) + if err != nil { + msg := `couldn't perform authorization checks` + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + return + } + if !ok { + msg := fmt.Sprintf(`you don't have permissions to create short filters in team [%s]`, team.ID.String()) + c.JSON(http.StatusForbidden, gin.H{"error": msg}) + return + } + + var payload filter.ShortFiltersPayload + if err := c.ShouldBindJSON(&payload); err != nil { + msg := `failed to parse filters json payload` + fmt.Println(msg, err) + c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + return + } + + shortFilters, err := filter.NewShortFilters(appId, payload.Filters) + if err != nil { + msg := `failed to create generate filter hash` + fmt.Println(msg, err) + c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + return + } + + shortFilters.Create() + + c.JSON(http.StatusOK, gin.H{"filter_short_code": shortFilters.Code}) +} diff --git a/backend/cleanup/cleanup/cleanup.go b/backend/cleanup/cleanup/cleanup.go index f5fb8a61f..02646770d 100644 --- a/backend/cleanup/cleanup/cleanup.go +++ b/backend/cleanup/cleanup/cleanup.go @@ -32,6 +32,10 @@ type StaleData struct { } func DeleteStaleData(ctx context.Context) { + // Delete shortened filters + deleteStaleShortenedFilters(ctx) + + // Delete events and attachments staleData, err := fetchStaleData(ctx) if err != nil { @@ -40,7 +44,6 @@ func DeleteStaleData(ctx context.Context) { } for _, st := range staleData { - // Delete attachments from object storage if len(st.Attachments) > 0 { fmt.Printf("Deleting %v attachments for app_id: %v\n", len(st.Attachments), st.AppID) @@ -73,7 +76,21 @@ func DeleteStaleData(ctx context.Context) { } staleDataJson, _ := json.MarshalIndent(staleData, "", " ") - fmt.Printf("Succesfully deleted stale data %v\n", string(staleDataJson)) + fmt.Printf("Succesfully deleted stale stale data %v\n", string(staleDataJson)) +} + +func deleteStaleShortenedFilters(ctx context.Context) { + threshold := time.Now().Add(-10 * time.Minute) // 10 min expiry + stmt := sqlf.PostgreSQL.DeleteFrom("public.short_filters"). + Where("created_at < ?", threshold) + + _, err := server.Server.PgPool.Exec(ctx, stmt.String(), stmt.Args()...) + if err != nil { + fmt.Printf("Failed to delete stale short filter codes: %v\n", err) + return + } + + fmt.Printf("Succesfully deleted stale short filters\n") } func fetchStaleData(ctx context.Context) ([]StaleData, error) { diff --git a/frontend/dashboard/app/api/api_calls.ts b/frontend/dashboard/app/api/api_calls.ts index 089044de5..2bc3e1d96 100644 --- a/frontend/dashboard/app/api/api_calls.ts +++ b/frontend/dashboard/app/api/api_calls.ts @@ -27,6 +27,12 @@ export enum FiltersApiStatus { NoData, Cancelled } +export enum SaveFiltersApiStatus { + Loading, + Success, + Error, + Cancelled +} export enum FiltersApiType { All, @@ -667,7 +673,56 @@ export class OsVersion { } } -function applyGenericFiltersToUrl(url: string, filters: Filters, keyId: string | null, keyTimestamp: string | null, limit: number | null) { +export const saveListFiltersToServer = async (filters: Filters) => { + if (filters.versions.length === 0 && + filters.osVersions.length === 0 && + filters.countries.length === 0 && + filters.networkProviders.length === 0 && + filters.networkTypes.length === 0 && + filters.networkGenerations.length === 0 && + filters.locales.length === 0 && + filters.deviceManufacturers.length === 0 && + filters.deviceNames.length === 0 + ) { + return null + } + + const origin = process.env.NEXT_PUBLIC_API_BASE_URL + let url = `${origin}/apps/${filters.app.id}/shortFilters` + const opts = { + method: 'POST', + body: JSON.stringify({ + filters: { + versions: filters.versions.map((v) => v.name), + version_codes: filters.versions.map((v) => v.code), + os_names: filters.osVersions.map((v) => v.name), + os_versions: filters.osVersions.map((v) => v.version), + countries: filters.countries, + network_providers: filters.networkProviders, + network_types: filters.networkTypes, + network_generations: filters.networkGenerations, + locales: filters.locales, + device_manufacturers: filters.deviceManufacturers, + device_names: filters.deviceNames + } + }) + } + + try { + const res = await fetchMeasure(url, opts); + + if (!res.ok) { + return null + } + + const data = await res.json() + return data.filter_short_code + } catch { + return null + } +} + +async function applyGenericFiltersToUrl(url: string, filters: Filters, keyId: string | null, keyTimestamp: string | null, limit: number | null) { const serverFormattedStartDate = formatUserInputDateToServerFormat(filters.startDate) const serverFormattedEndDate = formatUserInputDateToServerFormat(filters.endDate) const timezone = getTimeZoneForServer() @@ -679,51 +734,10 @@ function applyGenericFiltersToUrl(url: string, filters: Filters, keyId: string | searchParams.append('to', serverFormattedEndDate) searchParams.append('timezone', timezone) - // Append versions if present - if (filters.versions.length > 0) { - searchParams.append('versions', filters.versions.map(v => v.name).join(',')) - searchParams.append('version_codes', filters.versions.map(v => v.code).join(',')) - } - - // Append OS versions if present - if (filters.osVersions.length > 0) { - searchParams.append('os_names', filters.osVersions.map(v => v.name).join(',')) - searchParams.append('os_versions', filters.osVersions.map(v => v.version).join(',')) - } - - // Append countries if present - if (filters.countries.length > 0) { - searchParams.append('countries', filters.countries.join(',')) - } - - // Append network providers if present - if (filters.networkProviders.length > 0) { - searchParams.append('network_providers', filters.networkProviders.join(',')) - } - - // Append network types if present - if (filters.networkTypes.length > 0) { - searchParams.append('network_types', filters.networkTypes.join(',')) - } - - // Append network generations if present - if (filters.networkGenerations.length > 0) { - searchParams.append('network_generations', filters.networkGenerations.join(',')) - } - - // Append locales if present - if (filters.locales.length > 0) { - searchParams.append('locales', filters.locales.join(',')) - } - - // Append device manufacturers if present - if (filters.deviceManufacturers.length > 0) { - searchParams.append('device_manufacturers', filters.deviceManufacturers.join(',')) - } + const filterShortCode = await saveListFiltersToServer(filters) - // Append device names if present - if (filters.deviceNames.length > 0) { - searchParams.append('device_names', filters.deviceNames.join(',')) + if (filterShortCode !== null) { + searchParams.append('filter_short_code', filterShortCode) } // Append session type if needed @@ -858,7 +872,7 @@ export const fetchJourneyFromServer = async (journeyType: JourneyType, exception // Append bidirectional value url = url + `bigraph=${bidirectional ? '1&' : '0&'}` - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); @@ -881,7 +895,7 @@ export const fetchMetricsFromServer = async (filters: Filters, router: AppRouter let url = `${origin}/apps/${filters.app.id}/metrics?` - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); @@ -904,7 +918,7 @@ export const fetchSessionsOverviewFromServer = async (filters: Filters, keyId: s var url = `${origin}/apps/${filters.app.id}/sessions?` - url = applyGenericFiltersToUrl(url, filters, keyId, null, limit) + url = await applyGenericFiltersToUrl(url, filters, keyId, null, limit) try { const res = await fetchMeasure(url); @@ -927,7 +941,7 @@ export const fetchSessionsOverviewPlotFromServer = async (filters: Filters, rout var url = `${origin}/apps/${filters.app.id}/sessions/plots/instances?` - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); @@ -959,7 +973,7 @@ export const fetchExceptionsOverviewFromServer = async (exceptionsType: Exceptio url = `${origin}/apps/${filters.app.id}/anrGroups?` } - url = applyGenericFiltersToUrl(url, filters, keyId, null, limit) + url = await applyGenericFiltersToUrl(url, filters, keyId, null, limit) try { const res = await fetchMeasure(url); @@ -988,7 +1002,7 @@ export const fetchExceptionsDetailsFromServer = async (exceptionsType: Exception url = `${origin}/apps/${filters.app.id}/anrGroups/${exceptionsGroupdId}/anrs?` } - url = applyGenericFiltersToUrl(url, filters, keyId, keyTimestamp, limit) + url = await applyGenericFiltersToUrl(url, filters, keyId, keyTimestamp, limit) try { const res = await fetchMeasure(url); @@ -1017,7 +1031,7 @@ export const fetchExceptionsOverviewPlotFromServer = async (exceptionsType: Exce url = `${origin}/apps/${filters.app.id}/anrGroups/plots/instances?` } - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); @@ -1050,7 +1064,7 @@ export const fetchExceptionsDetailsPlotFromServer = async (exceptionsType: Excep url = `${origin}/apps/${filters.app.id}/anrGroups/${exceptionsGroupdId}/plots/instances?` } - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); @@ -1082,7 +1096,7 @@ export const fetchExceptionsDistributionPlotFromServer = async (exceptionsType: url = `${origin}/apps/${filters.app.id}/anrGroups/${exceptionsGroupdId}/plots/distribution?` } - url = applyGenericFiltersToUrl(url, filters, null, null, null) + url = await applyGenericFiltersToUrl(url, filters, null, null, null) try { const res = await fetchMeasure(url); diff --git a/frontend/dashboard/app/components/exceptions_details.tsx b/frontend/dashboard/app/components/exceptions_details.tsx index ca2d933a3..45e9e1fa5 100644 --- a/frontend/dashboard/app/components/exceptions_details.tsx +++ b/frontend/dashboard/app/components/exceptions_details.tsx @@ -124,8 +124,6 @@ export const ExceptionsDetails: React.FC = ({ exceptions {exceptionsDetailsApiStatus === ExceptionsDetailsApiStatus.Error &&

Error fetching list of {exceptionsType === ExceptionsType.Crash ? 'crashes' : 'ANRs'}, please change filters, refresh page or select a different app to try again

} - {exceptionsDetailsApiStatus === ExceptionsDetailsApiStatus.Success && exceptionsDetails.results === null &&

It seems there are no {exceptionsType === ExceptionsType.Crash ? 'Crashes' : 'ANRs'} for the current combination of filters. Please change filters to try again

} - {(exceptionsDetailsApiStatus === ExceptionsDetailsApiStatus.Success || exceptionsDetailsApiStatus === ExceptionsDetailsApiStatus.Loading) &&
@@ -146,7 +144,7 @@ export const ExceptionsDetails: React.FC = ({ exceptions {exceptionsDetailsApiStatus === ExceptionsDetailsApiStatus.Loading && } - {exceptionsDetails.results.length > 0 && + {exceptionsDetails.results?.length > 0 &&

Id: {exceptionsDetails.results[0].id}

Date & time: {formatDateToHumanReadableDateTime(exceptionsDetails.results[0].timestamp)}

diff --git a/frontend/dashboard/app/utils/auth/auth.ts b/frontend/dashboard/app/utils/auth/auth.ts index c743d444a..87ff098ff 100644 --- a/frontend/dashboard/app/utils/auth/auth.ts +++ b/frontend/dashboard/app/utils/auth/auth.ts @@ -380,9 +380,9 @@ export const fetchMeasure = (() => { // Get the base endpoint const endpoint = getEndpoint(resource); - // Cancel existing request for this endpoint if it exists + // Cancel existing request for this endpoint if it exists except in case of 'shortFilters' endpoint const existingController = inFlightRequests.get(endpoint); - if (existingController) { + if (existingController && !endpoint.includes('shortFilters')) { existingController.abort(); inFlightRequests.delete(endpoint); } diff --git a/self-host/postgres/20241102075104_create_short_filters_table.sql b/self-host/postgres/20241102075104_create_short_filters_table.sql new file mode 100644 index 000000000..7090aef34 --- /dev/null +++ b/self-host/postgres/20241102075104_create_short_filters_table.sql @@ -0,0 +1,17 @@ +-- migrate:up +create table if not exists public.short_filters ( + code varchar(32) primary key not null, + app_id uuid not null references public.apps(id) on delete cascade, + filters JSONB not null, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +comment on column public.short_filters.code is 'short code hashed from filters'; +comment on column public.short_filters.app_id is 'linked app id'; +comment on column public.short_filters.filters is 'filters JSON'; +comment on column public.short_filters.created_at is 'utc timestamp at the time of record creation'; +comment on column public.short_filters.updated_at is 'utc timestamp at the time of record update'; + +-- migrate:down +drop table if exists public.short_filters; \ No newline at end of file