From fa50eb75a00863ba02f771ab4e15d8933280b703 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 17:02:12 +0800 Subject: [PATCH] [#] lambda_worker: prefetching twitter OAuth key job --- cmd/lambda/main.go | 24 +- cmd/lambda_worker/main.go | 69 +-- go.mod | 16 +- go.sum | 27 +- util/s3/main.go | 47 ++ util/util.go | 18 + validator/twitter/api.go | 381 +--------------- validator/twitter/token.go | 418 ++++++++++++++++++ .../twitter/{api_test.go => token_test.go} | 6 +- 9 files changed, 578 insertions(+), 428 deletions(-) create mode 100644 util/s3/main.go create mode 100644 validator/twitter/token.go rename validator/twitter/{api_test.go => token_test.go} (93%) diff --git a/cmd/lambda/main.go b/cmd/lambda/main.go index 223d66d..4faf992 100644 --- a/cmd/lambda/main.go +++ b/cmd/lambda/main.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "os" "github.com/akrylysov/algnhsa" "github.com/aws/aws-sdk-go-v2/aws" @@ -13,6 +12,7 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/controller" "github.com/nextdotid/proof_server/model" + "github.com/nextdotid/proof_server/util" "github.com/nextdotid/proof_server/util/sqs" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" @@ -33,7 +33,8 @@ var ( ) func init_db(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate flag from ENV + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } func init_sqs(cfg aws.Config) { @@ -81,8 +82,8 @@ func init_config_from_aws_secret() { if initialized { return } - secret_name := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secret_name := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -116,18 +117,3 @@ func init_config_from_aws_secret() { } initialized = true } - -func getE(env_key, default_value string) string { - result := os.Getenv(env_key) - if len(result) == 0 { - if len(default_value) > 0 { - return default_value - } else { - logrus.Fatalf("ENV %s must be given! Abort.", env_key) - return "" - } - - } else { - return result - } -} diff --git a/cmd/lambda_worker/main.go b/cmd/lambda_worker/main.go index 36027f3..b3aa680 100644 --- a/cmd/lambda_worker/main.go +++ b/cmd/lambda_worker/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strconv" "github.com/aws/aws-lambda-go/events" @@ -20,6 +19,8 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/model" "github.com/nextdotid/proof_server/types" + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" "github.com/nextdotid/proof_server/validator/discord" @@ -38,6 +39,7 @@ import ( ) var ( + awsConfig aws.Config initialized = false wallet *goar.Wallet ) @@ -81,7 +83,7 @@ func handler(ctx context.Context, sqs_event events.SQSEvent) (events.SQSEventRes } case types.QueueActions.TwitterOAuthTokenAcquire: { - err := twitterRetrieveOAuthToken() + err := twitterRefreshOAuthToken() if err != nil { // Ignore errors for now fmt.Printf("Error when retrieving Twitter OAuth key: %s", err.Error()) @@ -134,7 +136,7 @@ func arweave_upload_many(personas []string) error { for _, pc := range chains { if pc.ArweaveID != "" { - continue + } previous, ok := lo.Find(chains, func(item *model.ProofChain) bool { @@ -241,8 +243,9 @@ func revalidateSingle(ctx context.Context, message *types.QueueMessage) error { return proof.Revalidate() } -func initDB(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate from ENV +func initDB() { + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } // func init_sqs(cfg aws.Config) { @@ -264,7 +267,8 @@ func initValidators() { } func init() { - cfg, err := config.LoadDefaultConfig( + var err error + awsConfig, err = config.LoadDefaultConfig( context.Background(), config.WithRegion("ap-east-1"), ) @@ -275,7 +279,7 @@ func init() { initConfigFromAWSSecret() logrus.SetLevel(logrus.InfoLevel) - initDB(cfg) + initDB() // init_sqs(cfg) initValidators() } @@ -284,8 +288,8 @@ func initConfigFromAWSSecret() { if initialized { return } - secretName := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secretName := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -327,33 +331,34 @@ func initConfigFromAWSSecret() { initialized = true } -func getE(envKey, defaultValue string) string { - result := os.Getenv(envKey) - if len(result) == 0 { - if len(defaultValue) > 0 { - return defaultValue - } else { - logrus.Fatalf("ENV %s must be given! Abort.", envKey) - return "" - } - - } else { - return result +func twitterRefreshOAuthToken() (err error) { + const VALID_TOKEN_AMOUNT = 5 + if err = utilS3.Init(awsConfig); err != nil { + logrus.Fatalf("Error during initializing S3 client: %s", err.Error()) } -} - -func twitterRetrieveOAuthToken() (err error) { - type TokenList struct { - Tokens []twitter.Tokens `json:"tokens"` + ctx := context.Background() + tokens, err := twitter.GetTokenListFromS3(ctx) + if err != nil { + logrus.Fatalf("Error when loading Twitter token list from S3: %s", err.Error()) } - // TODO: Retrieve existed token from a storage space (i.e., KV / S3) - tokens, err := twitter.GenerateOauthToken() - if err != nil { - return err + validTokens := lo.Filter(tokens.Tokens, func(token twitter.Token, _index int) bool { + return !token.IsExpired() + }) + if len(validTokens) < VALID_TOKEN_AMOUNT { + // Generate a new one + newToken, err := twitter.GenerateOauthToken() + if err != nil { + return err + } + fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) + validTokens = append(validTokens, *newToken) + newTokenList := twitter.TokenList{ + Tokens: validTokens, + } + newTokenListJSON, _ := newTokenList.ToJSON() + utilS3.PutToS3(ctx, twitter.TWITTER_TOKEN_LIST_FILENAME, newTokenListJSON) } - fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) - // TODO: save new token to a storage space (i.e., KV / S3) return nil } diff --git a/go.mod b/go.mod index 10e6e40..23ca0f8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/aws/aws-lambda-go v1.31.1 - github.com/aws/aws-sdk-go-v2 v1.16.5 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.15.4 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 github.com/everFinance/goar v1.4.2 @@ -26,15 +26,20 @@ require ( contrib.go.opencensus.io/exporter/stackdriver v0.13.4 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 // indirect - github.com/aws/smithy-go v1.11.3 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/blendle/zapdriver v1.3.1 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect @@ -148,6 +153,7 @@ require ( require ( github.com/akrylysov/algnhsa v0.12.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 github.com/bwmarrin/discordgo v0.25.0 github.com/ethereum/go-ethereum v1.10.25 diff --git a/go.sum b/go.sum index d9fe55d..c58018d 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,11 @@ github.com/aws/aws-lambda-go v1.31.1/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+K github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= github.com/aws/aws-sdk-go-v2/config v1.15.4 h1:P4mesY1hYUxru4f9SU0XxNKXmzfxsD0FtMIPRBjkH7Q= github.com/aws/aws-sdk-go-v2/config v1.15.4/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4= github.com/aws/aws-sdk-go-v2/credentials v1.12.0 h1:4R/NqlcRFSkR0wxOhgHi+agGpbEr5qMCjn7VqUIJY+E= @@ -84,15 +87,28 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/Cljk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 h1:FP8gquGeGHHdfY6G5llaMQDF+HAf20VKc8opRwmjf04= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 h1:6cZRymlLEIlDTEB0+5+An6Zj1CKt6rSE69tOmFeu1nk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 h1:m+mxqLIrGq7GJo5qw4rHn8BbUqHrvxvwFx54N1Pglvw= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6/go.mod h1:Z+i6uqZgCOBXhNoEGoRm/ZaLsaJA9rGUAmkVKM/3+g4= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 h1:HlEYt9p1TAQYxeB8jz3y4dmXmZevX+cJnh8OU6x0aqo= @@ -102,8 +118,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmd github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A= github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/util/s3/main.go b/util/s3/main.go new file mode 100644 index 0000000..fb9d88e --- /dev/null +++ b/util/s3/main.go @@ -0,0 +1,47 @@ +package s3 + +import ( + "bytes" + "context" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/nextdotid/proof_server/util" +) + +var ( + s3Client *s3.Client + bucketName string +) + +func Init(awsConfig aws.Config) error { + if s3Client == nil { + s3Client = s3.NewFromConfig(awsConfig) + } + bucketName = util.GetE("S3_BUCKET", "") + return nil +} + +// ReadFromS3 should be called before Init() +func ReadFromS3(ctx context.Context, key string) (content []byte, err error) { + result, err := s3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err != nil { + return []byte{}, err + } + defer result.Body.Close() + return io.ReadAll(result.Body) +} + +// PutToS3 should be called before Init() +func PutToS3(ctx context.Context, key string, content []byte) (err error) { + _, err = s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: bytes.NewReader(content), + }) + return err +} diff --git a/util/util.go b/util/util.go index 6a06e8e..cc5f5bc 100644 --- a/util/util.go +++ b/util/util.go @@ -2,10 +2,12 @@ package util import ( "encoding/base64" + "os" "strconv" "time" "github.com/nextdotid/proof_server/util/base1024" + "github.com/sirupsen/logrus" "golang.org/x/xerrors" ) @@ -30,3 +32,19 @@ func DecodeString(s string) ([]byte, error) { } return base1024.DecodeString(s) } + +// GetE gets current system's environment variable as `string`. +// If `defaultValue` is empty, this environment key must be exist, or it will panic. +func GetE(envKey, defaultValue string) string { + result := os.Getenv(envKey) + if len(result) == 0 { + if len(defaultValue) > 0 { + return defaultValue + } else { + logrus.Fatalf("ENV %s must be given! Abort.", envKey) + return "" + } + + } + return result +} diff --git a/validator/twitter/api.go b/validator/twitter/api.go index 4b2577f..12348a1 100644 --- a/validator/twitter/api.go +++ b/validator/twitter/api.go @@ -1,14 +1,9 @@ package twitter import ( - "encoding/json" - "fmt" - "io" - "net/http" - "strings" + "context" "time" - "github.com/nextdotid/proof_server/util" "github.com/samber/lo" "golang.org/x/xerrors" ) @@ -21,372 +16,30 @@ type APIResponse struct { Text string `json:"text"` } -type Tokens struct { - AccessToken string `json:"access_token"` - GuestToken string `json:"guest_token"` - FlowToken string `json:"flow_token"` - OAuthKey string `json:"oauth_key"` - OAuthSecret string `json:"oauth_secret"` - CreatedAt string `json:"created_at"` -} - -const ( - TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" - TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +var ( + CurrentTokenList *TokenList ) func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { const RETRY_AFTER = time.Second - - return nil, nil -} - -func setHeaders(req *http.Request, tokens *Tokens, setAccessToken, setGuestToken bool) { - req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("X-Twitter-API-Version", "5") - req.Header.Set("X-Twitter-Client", "TwitterAndroid") - req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") - req.Header.Set("OS-Version", "28") - req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") - req.Header.Set("X-Twitter-Active-User", "yes") - if setGuestToken { - req.Header.Set("X-Guest-Token", tokens.GuestToken) - } - if setAccessToken { - req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) - } -} - -// GenerateOauthToken generates a new Twitter OAuth guest token -// which can be used in calling Official APIs. -func GenerateOauthToken() (tokens *Tokens, err error) { - tokens = new(Tokens) - tokens.CreatedAt = util.TimeToTimestampString(time.Now()) - - if err := tokens.getFlowToken(); err != nil { - return nil, err - } - l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", tokens.AccessToken, tokens.GuestToken, tokens.FlowToken) - - requestBody := fmt.Sprintf(`{ - "flow_token": "%s", - "subtask_inputs": [ - { - "open_link": { - "link": "next_link" - }, - "subtask_id": "NextTaskOpenLink" - } - ], - "subtask_versions": { - "generic_urt": 3, - "standard": 1, - "open_home_timeline": 1, - "app_locale_update": 1, - "enter_date": 1, - "email_verification": 3, - "enter_password": 5, - "enter_text": 5, - "one_tap": 2, - "cta": 7, - "single_sign_on": 1, - "fetch_persisted_data": 1, - "enter_username": 3, - "web_modal": 2, - "fetch_temporary_password": 1, - "menu_dialog": 1, - "sign_up_review": 5, - "interest_picker": 4, - "user_recommendations_urt": 3, - "in_app_notification": 1, - "sign_up": 2, - "typeahead_search": 1, - "user_recommendations_list": 4, - "cta_inline": 1, - "contacts_live_sync_permission_prompt": 3, - "choice_selection": 5, - "js_instrumentation": 1, - "alert_dialog_suppress_client_events": 1, - "privacy_options": 1, - "topics_selector": 1, - "wait_spinner": 3, - "tweet_selection_urt": 1, - "end_flow": 1, - "settings_list": 7, - "open_external_link": 1, - "phone_verification": 5, - "security_key": 3, - "select_banner": 2, - "upload_media": 1, - "web": 2, - "alert_dialog": 1, - "open_account": 2, - "action_list": 2, - "enter_phone": 2, - "open_link": 1, - "show_code": 1, - "update_users": 1, - "check_logged_in_account": 1, - "enter_email": 2, - "select_avatar": 4, - "location_permission_prompt": 2, - "notifications_permission_prompt": 4 - } - }`, tokens.FlowToken) - - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) - if err != nil { - return nil, err - } - setHeaders(req, tokens, true, true) - - resp, err := new(http.Client).Do(req) - if err != nil { - return nil, err - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - l.Infof("Response: \n%s\n", body) - type ResponseSubtask struct { - // Should exist - OpenAccount *struct { - OauthToken string `json:"oauth_token"` - OauthTokenSecret string `json:"oauth_token_secret"` - } `json:"open_account"` - } - - type Response struct { - // Should be empty - Errors *[]struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"errors"` - // Should be "success" - Status string `json:"status"` - // A new flow token, usually ends with ":3" - FlowToken string `json:"flow_token"` - Subtasks []ResponseSubtask `json:"subtasks"` - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return nil, err - } - if response.Errors != nil { - return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) - } - if response.Status != "success" { - return nil, xerrors.Errorf("wrong API status: %s", response.Status) - } - - st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { - return (subtask.OpenAccount != nil) - }) - if !found { - return nil, xerrors.Errorf("oauth token not found in response") - } - // Update new FlowToken - tokens.FlowToken = response.FlowToken - l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) - - return tokens, nil -} - -func (tokens *Tokens) getFlowToken() (err error) { - if tokens.GuestToken == "" { - if err := tokens.getGuestToken(); err != nil { - return err + ctx := context.Background() + if CurrentTokenList == nil { + CurrentTokenList, err = GetTokenListFromS3(ctx) + if err != nil { + return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) } - } - - requestBody := `{ - "flow_token": null, - "input_flow_data": { - "country_code": null, - "flow_context": { - "start_location": { - "location": "splash_screen" - } - }, - "requested_variant": null, - "target_user_id": 0 - }, - "subtask_versions": { - "generic_urt": 3, - "standard": 1, - "open_home_timeline": 1, - "app_locale_update": 1, - "enter_date": 1, - "email_verification": 3, - "enter_password": 5, - "enter_text": 5, - "one_tap": 2, - "cta": 7, - "single_sign_on": 1, - "fetch_persisted_data": 1, - "enter_username": 3, - "web_modal": 2, - "fetch_temporary_password": 1, - "menu_dialog": 1, - "sign_up_review": 5, - "interest_picker": 4, - "user_recommendations_urt": 3, - "in_app_notification": 1, - "sign_up": 2, - "typeahead_search": 1, - "user_recommendations_list": 4, - "cta_inline": 1, - "contacts_live_sync_permission_prompt": 3, - "choice_selection": 5, - "js_instrumentation": 1, - "alert_dialog_suppress_client_events": 1, - "privacy_options": 1, - "topics_selector": 1, - "wait_spinner": 3, - "tweet_selection_urt": 1, - "end_flow": 1, - "settings_list": 7, - "open_external_link": 1, - "phone_verification": 5, - "security_key": 3, - "select_banner": 2, - "upload_media": 1, - "web": 2, - "alert_dialog": 1, - "open_account": 2, - "action_list": 2, - "enter_phone": 2, - "open_link": 1, - "show_code": 1, - "update_users": 1, - "check_logged_in_account": 1, - "enter_email": 2, - "select_avatar": 4, - "location_permission_prompt": 2, - "notifications_permission_prompt": 4 - } - }` - - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) - if err != nil { - return err - } - setHeaders(req, tokens, true, true) - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - type Response struct { - FlowToken string `json:"flow_token"` - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - - if response.FlowToken == "" { - return xerrors.Errorf("empty FlowToken") - } - - tokens.FlowToken = response.FlowToken - return nil -} - -func (tokens *Tokens) getGuestToken() (err error) { - if tokens.GuestToken != "" { - return nil - } - if tokens.AccessToken == "" { - if err = tokens.getAccessToken(); err != nil { - return err + if CurrentTokenList == nil { + return nil, xerrors.Errorf("twitter token list does not exist") } } - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) - if err != nil { - return err - } - setHeaders(req, tokens, true, false) - type Response struct { - GuestToken string `json:"guest_token"` - } - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - - // Fetching bearerToken - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - if response.GuestToken == "" { - return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + token := lo.Sample(CurrentTokenList.Tokens) + if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { + return nil, xerrors.Errorf("twitter token seems to be empty") } - tokens.GuestToken = response.GuestToken - return nil -} - -func (tokens *Tokens) getAccessToken() (err error) { - if tokens.AccessToken != "" { - return nil - } - - type Response struct { - TokenType string `json:"token_type"` - AccessToken string `json:"access_token"` - } - req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) - if err != nil { - return err - } - setHeaders(req, tokens, false, false) - req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } + // TODO: Use token to query specific tweet with twitter API + // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets + // https://api.twitter.com/1.1/statuses/show.json - // Fetching bearerToken - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - - if response.TokenType != "bearer" || len(response.AccessToken) == 0 { - return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) - } - - tokens.AccessToken = response.AccessToken - return nil -} - -func (tokens *Tokens) IsExpired() bool { - const EXPIRED_AT = "24h" - expiredAt, _ := time.ParseDuration(EXPIRED_AT) - createdAt, _ := util.TimestampStringToTime(tokens.CreatedAt) - return createdAt.Add(expiredAt).Before(time.Now()) + return nil, nil } diff --git a/validator/twitter/token.go b/validator/twitter/token.go new file mode 100644 index 0000000..a798dd8 --- /dev/null +++ b/validator/twitter/token.go @@ -0,0 +1,418 @@ +package twitter + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" + "github.com/samber/lo" + "golang.org/x/xerrors" +) + +const ( + TWITTER_TOKEN_LIST_FILENAME string = "config/proof_service/twitter_oauth_tokens.json" +) + +type Token struct { + AccessToken string `json:"access_token"` + GuestToken string `json:"guest_token"` + FlowToken string `json:"flow_token"` + OAuthKey string `json:"oauth_key"` + OAuthSecret string `json:"oauth_secret"` + CreatedAt string `json:"created_at"` +} + +type TokenList struct { + Tokens []Token `json:"tokens"` +} + +const ( + TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" + TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +) + +func setHeaders(req *http.Request, tokens *Token, setAccessToken, setGuestToken bool) { + req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Twitter-API-Version", "5") + req.Header.Set("X-Twitter-Client", "TwitterAndroid") + req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") + req.Header.Set("OS-Version", "28") + req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") + req.Header.Set("X-Twitter-Active-User", "yes") + if setGuestToken { + req.Header.Set("X-Guest-Token", tokens.GuestToken) + } + if setAccessToken { + req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) + } +} + +// GenerateOauthToken generates a new Twitter OAuth guest token +// which can be used in calling Official APIs. +func GenerateOauthToken() (token *Token, err error) { + token = new(Token) + token.CreatedAt = util.TimeToTimestampString(time.Now()) + + if err := token.getFlowToken(); err != nil { + return nil, err + } + l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", token.AccessToken, token.GuestToken, token.FlowToken) + + requestBody := fmt.Sprintf(`{ + "flow_token": "%s", + "subtask_inputs": [ + { + "open_link": { + "link": "next_link" + }, + "subtask_id": "NextTaskOpenLink" + } + ], + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }`, token.FlowToken) + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) + if err != nil { + return nil, err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + l.Infof("Response: \n%s\n", body) + type ResponseSubtask struct { + // Should exist + OpenAccount *struct { + OauthToken string `json:"oauth_token"` + OauthTokenSecret string `json:"oauth_token_secret"` + } `json:"open_account"` + } + + type Response struct { + // Should be empty + Errors *[]struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"errors"` + // Should be "success" + Status string `json:"status"` + // A new flow token, usually ends with ":3" + FlowToken string `json:"flow_token"` + Subtasks []ResponseSubtask `json:"subtasks"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return nil, err + } + if response.Errors != nil { + return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) + } + if response.Status != "success" { + return nil, xerrors.Errorf("wrong API status: %s", response.Status) + } + + st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { + return (subtask.OpenAccount != nil) + }) + if !found { + return nil, xerrors.Errorf("oauth token not found in response") + } + // Update new FlowToken + token.FlowToken = response.FlowToken + l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) + + return token, nil +} + +func (token *Token) getFlowToken() (err error) { + if token.GuestToken == "" { + if err := token.getGuestToken(); err != nil { + return err + } + } + + requestBody := `{ + "flow_token": null, + "input_flow_data": { + "country_code": null, + "flow_context": { + "start_location": { + "location": "splash_screen" + } + }, + "requested_variant": null, + "target_user_id": 0 + }, + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }` + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) + if err != nil { + return err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + type Response struct { + FlowToken string `json:"flow_token"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.FlowToken == "" { + return xerrors.Errorf("empty FlowToken") + } + + token.FlowToken = response.FlowToken + return nil +} + +func (token *Token) getGuestToken() (err error) { + if token.GuestToken != "" { + return nil + } + if token.AccessToken == "" { + if err = token.getAccessToken(); err != nil { + return err + } + } + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) + if err != nil { + return err + } + setHeaders(req, token, true, false) + type Response struct { + GuestToken string `json:"guest_token"` + } + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + if response.GuestToken == "" { + return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + } + token.GuestToken = response.GuestToken + + return nil +} + +func (token *Token) getAccessToken() (err error) { + if token.AccessToken != "" { + return nil + } + + type Response struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + } + req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) + if err != nil { + return err + } + setHeaders(req, token, false, false) + req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.TokenType != "bearer" || len(response.AccessToken) == 0 { + return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) + } + + token.AccessToken = response.AccessToken + return nil +} + +func (token *Token) IsExpired() bool { + const EXPIRED_AT = "24h" + expiredAt, _ := time.ParseDuration(EXPIRED_AT) + createdAt, _ := util.TimestampStringToTime(token.CreatedAt) + return createdAt.Add(expiredAt).Before(time.Now()) +} + +func (tl *TokenList) ToJSON() ([]byte, error) { + return json.Marshal(*tl) +} + +func TokenListFromJSON(content []byte) (*TokenList, error) { + tl := new(TokenList) + err := json.Unmarshal(content, tl) + if err != nil { + return nil, err + } + return tl, nil +} + +// GetTokenListFromS3 reads JSON file in AWS S3 to get token lists. +func GetTokenListFromS3(ctx context.Context) (*TokenList, error) { + body, _ := utilS3.ReadFromS3(ctx, TWITTER_TOKEN_LIST_FILENAME) + // If file not found, touch a new one: + if len(body) == 0 { + tl := new(TokenList) + newTokenList, _ := tl.ToJSON() + if err := utilS3.PutToS3(ctx, TWITTER_TOKEN_LIST_FILENAME, newTokenList); err != nil { + return nil, err + } + return tl, nil + } + + // File found, read it + return TokenListFromJSON(body) +} diff --git a/validator/twitter/api_test.go b/validator/twitter/token_test.go similarity index 93% rename from validator/twitter/api_test.go rename to validator/twitter/token_test.go index d7748ef..d671f37 100644 --- a/validator/twitter/api_test.go +++ b/validator/twitter/token_test.go @@ -8,7 +8,7 @@ import ( func Test_getAccessToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getAccessToken()) require.NotEmpty(t, tokens.AccessToken) t.Logf("Access token: %s", tokens.AccessToken) @@ -17,7 +17,7 @@ func Test_getAccessToken(t *testing.T) { func Test_getGuestToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getGuestToken()) require.NotEmpty(t, tokens.GuestToken) t.Logf("Guest token: %s", tokens.GuestToken) @@ -26,7 +26,7 @@ func Test_getGuestToken(t *testing.T) { func Test_FlowToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getFlowToken()) require.NotEmpty(t, tokens.FlowToken) t.Logf("Flow token: %s", tokens.FlowToken)