diff --git a/measure-backend/measure-go/event/attachment.go b/measure-backend/measure-go/event/attachment.go index cd42dff54..a65e3eb1a 100644 --- a/measure-backend/measure-go/event/attachment.go +++ b/measure-backend/measure-go/event/attachment.go @@ -4,6 +4,7 @@ import ( "errors" "io" "mime" + "net/url" "path/filepath" "slices" "time" @@ -57,7 +58,7 @@ func (a *Attachment) Upload() (output *s3manager.UploadOutput, err error) { // if a custom endpoint was set, then most likely, // we are in local development mode and should force - // path style instead of S3 virual path styles. + // path style instead of S3 virtual path styles. if config.AWSEndpoint != "" { awsConfig.S3ForcePathStyle = aws.Bool(true) awsConfig.Endpoint = aws.String(config.AWSEndpoint) @@ -94,12 +95,24 @@ func (a *Attachment) PreSignURL() (err error) { Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""), } + shouldProxy := true + + if config.AttachmentOrigin != "" { + shouldProxy = false + } + // if a custom endpoint was set, then most likely, - // we are in local development mode and should force - // path style instead of S3 virual path styles. + // external object store is not native S3 like, + // hence should force path style instead of S3 virtual + // path styles. if config.AWSEndpoint != "" { awsConfig.S3ForcePathStyle = aws.Bool(true) - awsConfig.Endpoint = aws.String(config.AttachmentOrigin) + + if shouldProxy { + awsConfig.Endpoint = aws.String(config.AWSEndpoint) + } else { + awsConfig.Endpoint = aws.String(config.AttachmentOrigin) + } } awsSession := session.Must(session.NewSession(awsConfig)) @@ -115,6 +128,37 @@ func (a *Attachment) PreSignURL() (err error) { return err } + if shouldProxy { + endpoint, err := url.JoinPath(config.APIOrigin, "attachments") + if err != nil { + return err + } + + proxyUrl, err := url.Parse(endpoint) + if err != nil { + return err + } + + parsed, err := url.Parse(urlStr) + if err != nil { + return err + } + + // clear the scheme and host of + // presigned URL, because we take interest + // in capturing the presigned URL's path + // and query string only. + parsed.Scheme = "" + parsed.Host = "" + + query := proxyUrl.Query() + + query.Set("payload", parsed.String()) + proxyUrl.RawQuery = query.Encode() + + urlStr = proxyUrl.String() + } + a.Location = urlStr return diff --git a/measure-backend/measure-go/main.go b/measure-backend/measure-go/main.go index 6938ee884..0d3ef00f0 100644 --- a/measure-backend/measure-go/main.go +++ b/measure-backend/measure-go/main.go @@ -69,8 +69,18 @@ func main() { c.String(http.StatusOK, "pong") }) - // Auth routes + // SDK routes + r.PUT("/events", measure.ValidateAPIKey(), measure.PutEvents) + r.PUT("/builds", measure.ValidateAPIKey(), measure.PutBuild) + + // Dashboard routes + // Any route below this point will use CORS r.Use(cors) + + // Proxy route + r.GET("/attachments", measure.ProxyAttachment) + + // Auth routes auth := r.Group("/auth") { auth.POST("github", measure.SigninGitHub) @@ -79,13 +89,7 @@ func main() { auth.DELETE("signout", measure.ValidateRefreshToken(), measure.Signout) } - // SDK routes - r.PUT("/events", measure.ValidateAPIKey(), measure.PutEvents) - r.PUT("/builds", measure.ValidateAPIKey(), measure.PutBuild) - - // Dashboard routes - r.Use(cors).Use(measure.ValidateAccessToken()) - apps := r.Group("/apps") + apps := r.Group("/apps", measure.ValidateAccessToken()) { apps.GET(":id/journey", measure.GetAppJourney) apps.GET(":id/metrics", measure.GetAppMetrics) @@ -107,7 +111,7 @@ func main() { apps.PATCH(":id/settings", measure.UpdateAppSettings) } - teams := r.Group("/teams") + teams := r.Group("/teams", measure.ValidateAccessToken()) { teams.POST("", measure.CreateTeam) teams.GET("", measure.GetTeams) diff --git a/measure-backend/measure-go/measure/app.go b/measure-backend/measure-go/measure/app.go index 91dab5929..feb3d0584 100644 --- a/measure-backend/measure-go/measure/app.go +++ b/measure-backend/measure-go/measure/app.go @@ -2551,8 +2551,7 @@ func GetCrashDetailCrashes(c *gin.Context) { return } - // generate pre-sign URLs for - // attachments + // set appropriate attachment URLs for i := range eventExceptions { if len(eventExceptions[i].Attachments) > 0 { for j := range eventExceptions[i].Attachments { @@ -3301,8 +3300,7 @@ func GetANRDetailANRs(c *gin.Context) { return } - // generate pre-sign URLs for - // attachments + // set appropriate attachment URLs for i := range eventANRs { if len(eventANRs[i].Attachments) > 0 { for j := range eventANRs[i].Attachments { diff --git a/measure-backend/measure-go/measure/attachment.go b/measure-backend/measure-go/measure/attachment.go new file mode 100644 index 000000000..86f747bf3 --- /dev/null +++ b/measure-backend/measure-go/measure/attachment.go @@ -0,0 +1,65 @@ +package measure + +import ( + "fmt" + "measure-backend/measure-go/server" + "net/http" + "net/http/httputil" + "net/url" + + "github.com/gin-gonic/gin" +) + +// ProxyAttachment proxies presigned S3 URLs to an S3-like +// server. +// +// We parse the payload from the incoming request's query +// string, then construct a new URL by replacing the S3 origin. +// Next, we create a reverse proxy and configure it to pipe +// response back to the original caller. +// +// The original S3 origin used when constructing the presigned +// URL must match the proxied presigned URL. +func ProxyAttachment(c *gin.Context) { + payload := c.Query("payload") + if payload == "" { + msg := `need payload for proxying to object store` + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) + return + } + + config := server.Server.Config + presignedUrl := config.AWSEndpoint + payload + + parsed, err := url.Parse(presignedUrl) + if err != nil { + msg := "failed to parse reconstructed presigned url" + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) + return + } + + proxy := httputil.NewSingleHostReverseProxy(parsed) + proxy.Director = func(req *http.Request) { + req.URL.Scheme = parsed.Scheme + req.URL.Host = parsed.Host + req.URL.Path = parsed.Path + req.URL.RawQuery = parsed.RawQuery + req.Host = parsed.Host + + req.Header = c.Request.Header + + fmt.Printf("Attachment proxy target url: %s\n", req.URL.String()) + } + + proxy.ModifyResponse = func(resp *http.Response) error { + fmt.Printf("Attachment proxy http status: %s\n", resp.Status) + return nil + } + + proxy.ServeHTTP(c.Writer, c.Request) +} diff --git a/measure-backend/measure-go/measure/mapping.go b/measure-backend/measure-go/measure/mapping.go index 79c33a9a2..05c40050c 100644 --- a/measure-backend/measure-go/measure/mapping.go +++ b/measure-backend/measure-go/measure/mapping.go @@ -184,7 +184,7 @@ func (bm *BuildMapping) upload() (*s3manager.UploadOutput, error) { // if a custom endpoint was set, then most likely, // we are in local development mode and should force - // path style instead of S3 virual path styles. + // path style instead of S3 virtual path styles. if config.AWSEndpoint != "" { awsConfig.S3ForcePathStyle = aws.Bool(true) awsConfig.Endpoint = aws.String(config.AWSEndpoint) diff --git a/measure-backend/measure-go/server/server.go b/measure-backend/measure-go/server/server.go index 968564fd7..87975448e 100644 --- a/measure-backend/measure-go/server/server.go +++ b/measure-backend/measure-go/server/server.go @@ -45,6 +45,7 @@ type ServerConfig struct { AWSEndpoint string AttachmentOrigin string SiteOrigin string + APIOrigin string OAuthGitHubKey string OAuthGitHubSecret string OAuthGoogleKey string @@ -102,7 +103,7 @@ func NewConfig() *ServerConfig { attachmentOrigin := os.Getenv("ATTACHMENTS_S3_ORIGIN") if attachmentOrigin == "" { - log.Println("ATTACHMENTS_S3_ORIGIN env var not set, event attachment downloads won't work") + log.Println("ATTACHMENTS_S3_ORIGIN env var not set, event attachment downloads will be proxied") } siteOrigin := os.Getenv("SITE_ORIGIN") @@ -110,6 +111,11 @@ func NewConfig() *ServerConfig { log.Fatal("SITE_ORIGIN env var not set. Need for Cross Origin Resource Sharing (CORS) to work.") } + apiOrigin := os.Getenv("API_ORIGIN") + if apiOrigin == "" { + log.Fatal("API_ORIGIN env var not set. Need for proxying session attachments.") + } + oauthGitHubKey := os.Getenv("OAUTH_GITHUB_KEY") if oauthGitHubKey == "" { log.Println("OAUTH_GITHUB_KEY env var is not set, dashboard authn won't work") @@ -171,6 +177,7 @@ func NewConfig() *ServerConfig { AWSEndpoint: endpoint, AttachmentOrigin: attachmentOrigin, SiteOrigin: siteOrigin, + APIOrigin: apiOrigin, OAuthGitHubKey: oauthGitHubKey, OAuthGitHubSecret: oauthGitHubSecret, OAuthGoogleKey: oauthGoogleKey, diff --git a/self-host/compose.yml b/self-host/compose.yml index 42ad2677e..f6e6dbe84 100644 --- a/self-host/compose.yml +++ b/self-host/compose.yml @@ -1,12 +1,30 @@ # Measure Docker Compose # # Guidelines +# # Follow these guidelines for a consistently maintained # compose file and a pleasurable maintainer experience. # # - Order first-party services at the top followed by # third-party services. Databases and other external # services should appear after measure services. +# - All environment variables MUST be passed using the +# `environment` directive of each service. That's the +# ONLY source of truth. +# +# Tips +# +# - For setting default values for any environment variable +# Specify the default value after the hyphen (-). To set +# empty default value do not specify anything after the +# hyphen. See example below. +# +# ```yaml +# environment: +# - MY_VAR=${MY_VAR:-default_value} +# - EMPTY_MY_VAR=${EMPTY_MY_VAR:-} +# ``` + name: measure services: @@ -46,12 +64,13 @@ services: - SYMBOLS_ACCESS_KEY=${SYMBOLS_ACCESS_KEY} - SYMBOLS_SECRET_ACCESS_KEY=${SYMBOLS_SECRET_ACCESS_KEY} - SYMBOLICATOR_ORIGIN=${SYMBOLICATOR_ORIGIN} - - ATTACHMENTS_S3_ORIGIN=${ATTACHMENTS_S3_ORIGIN} + - ATTACHMENTS_S3_ORIGIN=${ATTACHMENTS_S3_ORIGIN:-} - ATTACHMENTS_S3_BUCKET=${ATTACHMENTS_S3_BUCKET} - ATTACHMENTS_S3_BUCKET_REGION=${ATTACHMENTS_S3_BUCKET_REGION} - ATTACHMENTS_ACCESS_KEY=${ATTACHMENTS_ACCESS_KEY} - ATTACHMENTS_SECRET_ACCESS_KEY=${ATTACHMENTS_SECRET_ACCESS_KEY} - SITE_ORIGIN=${NEXT_PUBLIC_SITE_URL} + - API_ORIGIN=${NEXT_PUBLIC_API_BASE_URL} - OAUTH_GOOGLE_KEY=${OAUTH_GOOGLE_KEY} - OAUTH_GITHUB_KEY=${OAUTH_GITHUB_KEY} - OAUTH_GITHUB_SECRET=${OAUTH_GITHUB_SECRET} diff --git a/self-host/config.sh b/self-host/config.sh index a0b73b0d1..33e0325f3 100755 --- a/self-host/config.sh +++ b/self-host/config.sh @@ -193,7 +193,7 @@ SYMBOLS_ACCESS_KEY=minio SYMBOLS_SECRET_ACCESS_KEY=minio123 # Session attachments won't work without these -ATTACHMENTS_S3_ORIGIN=http://localhost:9119 +ATTACHMENTS_S3_ORIGIN= ATTACHMENTS_S3_BUCKET=msr-attachments-sandbox ATTACHMENTS_S3_BUCKET_REGION=us-east-1 ATTACHMENTS_ACCESS_KEY=minio @@ -488,7 +488,7 @@ elif [[ "$SETUP_ENV" == "production" ]]; then echo -e "\nSet storage bucket for attachments" echo -e "Example: https://measure-attachments.yourcompany.com" - ATTACHMENTS_S3_ORIGIN=$(prompt_value_manual "Enter attachments S3 bucket origin: ") + ATTACHMENTS_S3_ORIGIN="" ATTACHMENTS_S3_BUCKET="msr-$NAMESPACE-attachments" ATTACHMENTS_S3_BUCKET_REGION="us-east-1" ATTACHMENTS_ACCESS_KEY=$MINIO_ROOT_USER