diff --git a/CHANGELOG.md b/CHANGELOG.md index 15e975c..a95a8cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 1.5.0 - 2024-12-16 +* Add `ScanReader` for scanning a well-implemented `AmaasClientReader` +* Add example code `scan-s3obj` for scanning an S3 object which is an example of using `ScanReader` + ## 1.4.2 - 2024-08-30 * Fixed the issue of the TLS parameter being overwritten diff --git a/README.md b/README.md index fe987ab..2308eff 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,50 @@ if err != nil { // Use the 'response' as needed ``` +### Scanning with AmaasClientReader + +```go +type CustomReader struct { + ... +} + +func newCustomReader() *CustomReader { + ... +} + +func (r *CustomReader) Identifier() string { + // It returns the name of the file. +} + +func (r *CustomReader) DataSize() (int64, error) { + // It should return the true size of the file in Reader. +} + +func (r *CustomReader) ReadBytes(offset int64, length int32) (data []byte, err error) { + // It should return required number of data bytes starting from certain offset. +} + +reader := newCustomReader() + +// It is recommended to disable digest when using AmaasReader. +// Because it will trigger ReadBytes to read whole file, +// network traffic will increase if it reads from the Internet. +client.SetDigestDisable() + +response, err := client.ScanReader(reader, tags) +if err != nil { + // Handle scanning error + panic(err) +} + +// Use the 'response' as needed +``` + **_Note_** - Max number of tags is 8. And the length of each tag can't exceed 63. +- If user wants to take a look how to scan a S3 file without downloading the whole to the ground, + please refer to the [example code](examples/scan-s3obj/scan-s3obj.go) for further detail. ## Additional Functions diff --git a/VERSION b/VERSION index 9df886c..bc80560 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.2 +1.5.0 diff --git a/examples/scan-s3obj/go.mod b/examples/scan-s3obj/go.mod new file mode 100644 index 0000000..9357522 --- /dev/null +++ b/examples/scan-s3obj/go.mod @@ -0,0 +1,36 @@ +module github.com/trendmicro/tm-v1-fs-golang-sdk/examples/scan-s3obj + +go 1.23 + +require ( + github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/smithy-go v1.22.1 // indirect + github.com/golang/protobuf v1.5.4 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect +) + +require ( + github.com/trendmicro/tm-v1-fs-golang-sdk v0.0.0-20241119105152-8e5832d37e21 + github.com/aws/aws-sdk-go-v2/config v1.28.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1 +) diff --git a/examples/scan-s3obj/go.sum b/examples/scan-s3obj/go.sum new file mode 100644 index 0000000..d7decb9 --- /dev/null +++ b/examples/scan-s3obj/go.sum @@ -0,0 +1,66 @@ +github.com/trendmicro/tm-v1-fs-golang-sdk v0.0.0-20241119105152-8e5832d37e21 h1:30XGBlE8B4nrYGFPNxz8Q+tIPQPb95zKIHqmVM5jh8w= +github.com/trendmicro/tm-v1-fs-golang-sdk v0.0.0-20241119105152-8e5832d37e21/go.mod h1:P/bveu/shq7hy5xRt2h6L1H6yR2FGqaNIJ7lm+yJLKU= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= +github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24 h1:JX70yGKLj25+lMC5Yyh8wBtvB01GDilyRuJvXJ4piD0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24/go.mod h1:+Ln60j9SUTD0LEwnhEB0Xhg61DHqplBrbZpLgyjoEHg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5 h1:gvZOjQKPxFXy1ft3QnEyXmT+IqneM9QAUWlM3r0mfqw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5/go.mod h1:DLWnfvIcm9IET/mmjdxeXbBKmTCm0ZB8p1za9BVteM8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5 h1:P1doBzv5VEg1ONxnJss1Kh5ZG/ewoIE4MQtKKc6Crgg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5/go.mod h1:NOP+euMW7W3Ukt28tAxPuoWao4rhhqJD3QEBk7oCg7w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1 h1:LXLnDfjT/P6SPIaCE86xCOjJROPn4FNB2EdN68vMK5c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1/go.mod h1:ralv4XawHjEMaHOWnTFushl0WRqim/gQWesAMF6hTow= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/scan-s3obj/scan-s3obj.go b/examples/scan-s3obj/scan-s3obj.go new file mode 100644 index 0000000..ed25edb --- /dev/null +++ b/examples/scan-s3obj/scan-s3obj.go @@ -0,0 +1,183 @@ +package main + +/* +This is an example of a test program that can execute a scan on a HTTP URL. +It will return an array of AMaaS scan results as part of its JSON output. +*/ + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "os" + "strings" + + amaasclient "github.com/trendmicro/tm-v1-fs-golang-sdk" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +func main() { + + var bucketregion string + var bucket string + var key string + + var grpcAddr string + var apiKey string + var tls bool + var caCert string + var region string + var pml bool + var feedback bool + var verbose bool + var tag string + var digest bool + + flag.StringVar(&bucketregion, "bucketregion", "us-west-2", "region for S3 bucket") + flag.StringVar(&bucket, "bucket", "", "S3 bucket name") + flag.StringVar(&key, "key", "", "S3 object key") + + flag.StringVar(&grpcAddr, "addr", "", "the address to connect to for GRPC") + flag.StringVar(&apiKey, "apikey", "", "API key for service authentication") + flag.BoolVar(&tls, "tls", false, "enable server TLS by client for GRPC.") + flag.StringVar(®ion, "region", "", "the region to connect to") + flag.BoolVar(&pml, "pml", false, "enable predictive machine learning detection") + flag.BoolVar(&feedback, "feedback", false, "enable SPN feedback") + flag.BoolVar(&verbose, "verbose", false, "enable verbose scan result") + flag.StringVar(&tag, "tag", "", "tags to be used for scanning. separated by comma.") + flag.StringVar(&caCert, "ca_cert", "", "CA certificate for self hosted AMaaS server") + flag.BoolVar(&digest, "digest", false, "enable digest calculation. it might increase network traffic for cloud file.") + + flag.Parse() + + var ac *amaasclient.AmaasClient + var err error + + if region != "" && grpcAddr != "" { + log.Fatal("Both region and addr are specified. Please specify only one.") + } else if region != "" { + ac, err = amaasclient.NewClient(apiKey, region) + if err != nil { + log.Fatalf("Unable to create AMaaS scan client object. error: %v", err) + } + } else if grpcAddr != "" { + ac, err = amaasclient.NewClientInternal(apiKey, grpcAddr, tls, caCert) + if err != nil { + log.Fatalf("Unable to create AMaaS scan client object. error: %v", err) + } + } else { + log.Fatal("Neither region nor addr is specified. Please specify one.") + } + + if pml { + ac.SetPMLEnable() + } + + if feedback { + ac.SetFeedbackEnable() + } + + if verbose { + ac.SetVerboseEnable() + } + + if !digest { + // disable digest calculation to reduce network traffic if file is on cloud + ac.SetDigestDisable() + } + + var tagsArray []string + if tag != "" { + tagsArray = strings.Split(tag, ",") + } + + reader, err := NewS3ClientReader(context.Background(), bucketregion, bucket, key) + if err != nil { + log.Fatalf("Unable to create S3 client reader. error: %v", err) + } + + result, err := ac.ScanReader(reader, tagsArray) + if err != nil { + log.Fatalf("Unable to scan reader. error: %v", err) + } + + fmt.Printf("%v", result) + + os.Exit(0) +} + +type S3ClientReader struct { + client *s3.Client + bucket string + key string + size int64 +} + +func NewS3ClientReader(ctx context.Context, bucketregion, bucket, key string) (*S3ClientReader, error) { + // load default config from environment with specified region + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(bucketregion)) + if err != nil { + return nil, err + } + defer ctx.Done() + + // create S3 client with given config + client := s3.NewFromConfig(cfg) + + attr, err := client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &key, + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesObjectSize, + }, + }) + if err != nil { + return nil, err + } + + if attr.ObjectSize == nil { + return nil, fmt.Errorf("unable to get object size from S3") + } + + return &S3ClientReader{ + client: client, + bucket: bucket, + key: key, + size: *attr.ObjectSize, + }, nil +} + +// S3ClientReader implements AmaasClientReader +func (r *S3ClientReader) Identifier() string { + return fmt.Sprintf("s3://%s/%s", r.bucket, r.key) +} + +func (r *S3ClientReader) DataSize() (int64, error) { + return r.size, nil +} + +func (r *S3ClientReader) ReadBytes(offset int64, length int32) ([]byte, error) { + var rng string = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) + + output, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: &r.bucket, + Key: &r.key, + Range: &rng, + }) + if err != nil { + return nil, err + } + defer output.Body.Close() + + bytes, err := io.ReadAll(output.Body) + if err != nil && err != io.EOF { + bytes = nil + } + + return bytes, err +} diff --git a/grpc.go b/grpc.go index 473aca2..91d9143 100644 --- a/grpc.go +++ b/grpc.go @@ -9,9 +9,6 @@ import ( "encoding/hex" "errors" "fmt" - "golang.org/x/net/http/httpproxy" - "golang.org/x/net/proxy" - "google.golang.org/grpc/credentials/insecure" "hash" "io" "log" @@ -22,6 +19,10 @@ import ( "strings" "time" + "golang.org/x/net/http/httpproxy" + "golang.org/x/net/proxy" + "google.golang.org/grpc/credentials/insecure" + "golang.org/x/exp/slices" "google.golang.org/grpc" @@ -45,6 +46,7 @@ const ( maxTagsListSize = 8 maxTagSize = 63 + maxBatchSize = int32(1024 * 1024) ) type LogLevel int @@ -59,14 +61,6 @@ var userLogger LoggerCallback = nil // ///////////////////////////////////////////////// -type AmaasClientReader interface { - Identifier() string - DataSize() (int64, error) - ReadBytes(offset int64, length int32) ([]byte, error) - Close() - Hash(algorithm string) (string, error) -} - // File reader implementation type AmaasClientFileReader struct { @@ -240,6 +234,49 @@ type AmaasClient struct { digest bool } +func getHashValue(dataReader AmaasClientReader) (string, string, error) { + var offset, length int64 + + length, err := dataReader.DataSize() + if err != nil { + return "", "", err + } + + sha1 := sha1.New() + sha256 := sha256.New() + + for length > 0 { + var batch_size = maxBatchSize + + if int64(batch_size) > length { + batch_size = int32(length) + } + + bytes, err := dataReader.ReadBytes(offset, batch_size) + if err != nil || int32(len(bytes)) != batch_size { + return "", "", err + } + + _, err = sha1.Write(bytes) + if err != nil { + return "", "", err + } + + _, err = sha256.Write(bytes) + if err != nil { + return "", "", err + } + + offset += int64(batch_size) + length -= int64(batch_size) + } + + var s_sha1 = fmt.Sprintf("sha1:%s", hex.EncodeToString(sha1.Sum(nil))) + var s_sha256 = fmt.Sprintf("sha256:%s", hex.EncodeToString(sha256.Sum(nil))) + + return s_sha1, s_sha256, nil +} + func scanRun(ctx context.Context, cancel context.CancelFunc, c pb.ScanClient, dataReader AmaasClientReader, tags []string, pml bool, bulk bool, feedback bool, verbose bool, digest bool) (string, error) { @@ -275,8 +312,7 @@ func scanRun(ctx context.Context, cancel context.CancelFunc, c pb.ScanClient, da size, _ := dataReader.DataSize() if digest { - hashSha256, _ = dataReader.Hash("sha256") - hashSha1, _ = dataReader.Hash("sha1") + hashSha1, hashSha256, _ = getHashValue(dataReader) } if err = runInitRequest(stream, dataReader.Identifier(), uint64(size), hashSha256, hashSha1, tags, pml, bulk, feedback, @@ -462,6 +498,22 @@ func (ac *AmaasClient) fileScanRunNormalFile(fileName string, tags []string) (st ac.verbose, ac.digest) } +func (ac *AmaasClient) readerScanRun(reader AmaasClientReader, tags []string) (string, error) { + + if ac.conn == nil { + return "", makeInternalError(MSG("MSG_ID_ERR_CLIENT_NOT_READY")) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(ac.timeoutSecs)) + + ctx = ac.buildAuthContext(ctx) + + ctx = ac.buildAppNameContext(ctx) + + return scanRun(ctx, cancel, pb.NewScanClient(ac.conn), reader, tags, ac.pml, true, ac.feedback, + ac.verbose, ac.digest) +} + // Function to load TLS credentials with optional certificate verification func loadTLSCredentials(caCertPath string, verifyCert bool) (credentials.TransportCredentials, error) { logMsg(LogLevelDebug, "log TLS certificate = %s cert verify = %t", caCertPath, verifyCert) diff --git a/grpc_client_test.go b/grpc_client_test.go index 1b35925..19d2ee2 100644 --- a/grpc_client_test.go +++ b/grpc_client_test.go @@ -495,6 +495,84 @@ func TestFileClientSha256WithUnsupportedAlgo(t *testing.T) { assert.Error(t, err, "Unsupported algorithm for calculating the hash: sha224") } +type MockAmassClientReader struct { + size int64 +} + +func NewMockAmassClientReader(size int64) *MockAmassClientReader { + return &MockAmassClientReader{size: size} +} + +func (m *MockAmassClientReader) Identifier() string { + return "MockAmassClientReader" +} + +func (m *MockAmassClientReader) DataSize() (int64, error) { + return m.size, nil +} + +func (m *MockAmassClientReader) ReadBytes(offset int64, length int32) ([]byte, error) { + if offset < 0 || offset >= m.size { + return nil, fmt.Errorf("Invalid offset: %d", offset) + } + + // always return an array filled with zero + return make([]byte, length), nil +} + +func TestGetHashValue(t *testing.T) { + + reader := NewMockAmassClientReader(1024*1024 - 301) + sha1, sha256, _ := getHashValue(reader) + assert.Equal(t, "sha1:5e944e68476189d048e74673b02a62dc42118cc8", sha1) + assert.Equal(t, "sha256:5746a33a622bbfd8d32a6398bbc04e90b3269787b17a9740b95e04a641a47935", sha256) + + reader = NewMockAmassClientReader(1024*1024 - 1) + sha1, sha256, _ = getHashValue(reader) + assert.Equal(t, "sha1:24f30d3b09e9056c6b9f6dfd6f386c6828fd63c3", sha1) + assert.Equal(t, "sha256:ca7ed0c4a8e67cbdc461c4cb0d286d2fabbd9f0c41a7f42b665f72ebaa8aec56", sha256) + + reader = NewMockAmassClientReader(1024*1024 + 0) + sha1, sha256, _ = getHashValue(reader) + assert.Equal(t, "sha1:3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3", sha1) + assert.Equal(t, "sha256:30e14955ebf1352266dc2ff8067e68104607e750abb9d3b36582b8af909fcb58", sha256) + + reader = NewMockAmassClientReader(1024*1024 + 1) + sha1, sha256, _ = getHashValue(reader) + assert.Equal(t, "sha1:a84d35eda74338bd79a432f77d73f8ab5eb91902", sha1) + assert.Equal(t, "sha256:2cb74edba754a81d121c9db6833704a8e7d417e5b13d1a19f4a52f007d644264", sha256) + + reader = NewMockAmassClientReader(1024*1024 + 244) + sha1, sha256, _ = getHashValue(reader) + assert.Equal(t, "sha1:5f054033c49f65dce7ca3d30519663db572b040c", sha1) + assert.Equal(t, "sha256:e8b900132db114bbe70361feb988e5f3b59c88f936b3e8fe924d448f0689f42c", sha256) +} + +func TestGetHashValueWithBufferReader(t *testing.T) { + data := []byte("dummy") + reader, _ := InitBufferReader(data, "test") + sha1, sha256, _ := getHashValue(reader) + assert.Equal(t, "sha1:829c3804401b0727f70f73d4415e162400cbe57b", sha1) + assert.Equal(t, "sha256:b5a2c96250612366ea272ffac6d9744aaf4b45aacd96aa7cfcb931ee3b558259", sha256) + + data = []byte("dummy1234567890") + reader, _ = InitBufferReader(data, "test") + sha1, sha256, _ = getHashValue(reader) + assert.Equal(t, "sha1:84901b8dc51ff505905443f863d6b6e8e6eca1f3", sha1) + assert.Equal(t, "sha256:e47a0a4d0e7da5ab6a5e331a0400157b09c44926224e9357308805ade4ae8141", sha256) +} + +func TestGetHashValueWithFileReader(t *testing.T) { + dat := createTestDat("test.*.dat") + assert.NotNil(t, dat) + defer os.Remove(dat.Filename()) + + reader, _ := InitFileReader(dat.Filename()) + sha1, sha256, _ := getHashValue(reader) + assert.Equal(t, dat.Sha1(), sha1) + assert.Equal(t, dat.Sha256(), sha256) +} + func TestCheckAuthKey(t *testing.T) { tcs := []struct { testCase string diff --git a/grpc_run_test.go b/grpc_run_test.go index d6ad330..5b14a80 100644 --- a/grpc_run_test.go +++ b/grpc_run_test.go @@ -2,7 +2,11 @@ package client import ( "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" "errors" + "fmt" "math/rand" "os" "testing" @@ -43,6 +47,8 @@ const ( type TestDat struct { filesize int filename string + sha1 string + sha256 string } func createTestDat(fnameTemplate string) *TestDat { @@ -73,6 +79,15 @@ func createTestDat(fnameTemplate string) *TestDat { b[i] = byte(i % 256) } + sha1 := sha1.New() + sha1.Write(b) + + sha256 := sha256.New() + sha256.Write(b) + + d.sha1 = fmt.Sprintf("sha1:%s", hex.EncodeToString(sha1.Sum(nil))) + d.sha256 = fmt.Sprintf("sha256:%s", hex.EncodeToString(sha256.Sum(nil))) + if n, err := fd.Write(b); err != nil || n != d.filesize { return nil } @@ -92,6 +107,14 @@ func (d *TestDat) Filesize() int { return d.filesize } +func (d *TestDat) Sha1() string { + return d.sha1 +} + +func (d *TestDat) Sha256() string { + return d.sha256 +} + // Test readFileBytes() function which is critical to client retrieval // of file content. diff --git a/sdk.go b/sdk.go index 1f53075..f99552f 100644 --- a/sdk.go +++ b/sdk.go @@ -27,6 +27,17 @@ const ( LogLevelDebug LogLevel = 5 ) +type AmaasClientReader interface { + // Return the identifier of the data source. For example, name of the file being read. + Identifier() string + + // Return the total size of the data source. + DataSize() (int64, error) + + // Return requested number of bytes from the data source starting at the specified offset. + ReadBytes(offset int64, length int32) ([]byte, error) +} + func NewClient(key string, region string) (c *AmaasClient, e error) { ac := &AmaasClient{digest: true} @@ -76,8 +87,14 @@ func (ac *AmaasClient) Destroy() { } // -// The ScanFile() and ScanBuffer() functions are thread-safe and can be invoked by multiple threads, +// The ScanFile(), ScanBuffer() and ScanReader() functions are thread-safe and can be invoked by multiple threads, // but all file scans must complete before Destroy() can be invoked. + +// If your application utilizes ScanReader() for scanning, it's advisable to deactivate digest feature by SetDigestDisable(). +// This is because the Digest feature requires the entire file to be read/feteched. +// +// For instance, scanning a cloud-based file requires downloading the entire file locally for digest calculation. +// It increases network usage and processing time. // func (ac *AmaasClient) ScanFile(filePath string, tags []string) (resp string, e error) { @@ -90,6 +107,11 @@ func (ac *AmaasClient) ScanBuffer(buffer []byte, identifier string, tags []strin return ac.bufferScanRun(buffer, identifier, tags) } +func (ac *AmaasClient) ScanReader(reader AmaasClientReader, tags []string) (resp string, e error) { + currentLogLevel = getLogLevel() + return ac.readerScanRun(reader, tags) +} + func (ac *AmaasClient) DumpConfig() (output string) { return fmt.Sprintf("%+v", ac) }