Skip to content

Commit 47a7da4

Browse files
abhisekabhisek
committed
Added redis persistence module
1 parent 1dc4cfc commit 47a7da4

7 files changed

+99
-7
lines changed

‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ Two containers are used to deliver the service:
6565

6666
In memory data structure with [go-routine](https://tour.golang.org/concurrency/1) based asynchronous tasks. Check [service.go](https://github.com/abhisek/container-image-scanner-api/blob/master/service.go) for more details.
6767

68+
> Redis is used for persistence with *15 minutes* expiration for reports
69+
6870
## Proposed Architecture
6971

7072
![](docs/diagram.png)

‎docker-compose.yml

+8-1
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,17 @@ services:
88
- common:/var/run
99
ports:
1010
- "8000:8000"
11+
environment:
12+
- REDIS_ENDPOINT=redis:6379
1113
docker:
1214
image: docker:dind
1315
privileged: true
1416
volumes:
1517
- common:/var/run
18+
redis:
19+
image: redis:5.0-alpine
20+
volumes:
21+
- redis:/data
1622
volumes:
17-
common:
23+
common:
24+
redis:

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/aquasecurity/trivy v0.4.3
88
github.com/docker/distribution v2.7.1+incompatible
99
github.com/docker/docker v0.7.3-0.20190506211059-b20a14b54661
10+
github.com/go-redis/redis/v7 v7.0.0-beta.5
1011
github.com/go-stack/stack v1.8.0 // indirect
1112
github.com/google/uuid v1.1.1
1213
github.com/gorilla/handlers v1.4.2

‎go.sum

+9
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,10 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
110110
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
111111
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
112112
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
113+
github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
113114
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
115+
github.com/go-redis/redis/v7 v7.0.0-beta.5 h1:7bdbDkv2nKZm6Tydrvmay3xOvVaxpAT4ZsNTrSDMZUE=
116+
github.com/go-redis/redis/v7 v7.0.0-beta.5/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
114117
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
115118
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
116119
github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM=
@@ -216,9 +219,11 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190607075207-195002e6e56a/go.mod h1
216219
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
217220
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
218221
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
222+
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
219223
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
220224
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
221225
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
226+
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
222227
github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 h1:QhPf3A2AZW3tTGvHPg0TA+CR3oHbVLlXUhlghqISp1I=
223228
github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
224229
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
@@ -332,6 +337,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
332337
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
333338
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
334339
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
340+
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
335341
golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c h1:SRpq/kuj/xNci/RdvEs+RSvpfxqvLAzTKuKGlzoGdZQ=
336342
golang.org/x/net v0.0.0-20191108221443-4ba9e2ef068c/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
337343
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -364,11 +370,13 @@ golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX
364370
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
365371
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
366372
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
373+
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
367374
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
368375
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
369376
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
370377
golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
371378
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
379+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
372380
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
373381
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
374382
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -400,6 +408,7 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4
400408
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
401409
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
402410
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
411+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
403412
gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
404413
gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
405414
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

‎main.go

-2
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ func scanSubmissionHandler(w http.ResponseWriter, r *http.Request) {
6565
if err := decoder.Decode(&scanRequest); err != nil {
6666
json.NewEncoder(w).Encode(map[string]string{"error": "Failed to decode request params"})
6767
} else {
68-
// scanReport, _ := scanningService.ScanImage(scanRequest)
69-
// json.NewEncoder(w).Encode(scanReport)
7068
scanID := scanningService.AsyncScanImage(scanRequest)
7169
json.NewEncoder(w).Encode(map[string]string{"scan_id": scanID})
7270
}

‎persistence.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
"time"
8+
9+
"github.com/go-redis/redis/v7"
10+
log "github.com/sirupsen/logrus"
11+
)
12+
13+
type RedisPersistence struct {
14+
redis *redis.Client
15+
}
16+
17+
var DEFAULT_EXPIRATION time.Duration = 15 * time.Minute
18+
19+
func (client *RedisPersistence) Init() {
20+
log.Debugf("Initializing Redis client")
21+
22+
client.redis = redis.NewClient(&redis.Options{
23+
Addr: os.Getenv("REDIS_ENDPOINT"),
24+
Password: os.Getenv("REDIS_PASSWORD"),
25+
DB: 0})
26+
27+
pong, err := client.redis.Ping().Result()
28+
if err != nil {
29+
panic(err)
30+
}
31+
32+
log.Debugf("Redis client initialized: %s", pong)
33+
}
34+
35+
func (client *RedisPersistence) SetScanStatus(scanID, status string) error {
36+
key := fmt.Sprintf("scans:%s:status", scanID)
37+
_, err := client.redis.Set(key, status, 0).Result()
38+
39+
if err != nil {
40+
log.Debugf("Failed to set status map in Redis: %#v", err)
41+
}
42+
43+
return err
44+
}
45+
46+
func (client *RedisPersistence) SetScanReport(scanID string, report ScanReport) error {
47+
key := fmt.Sprintf("scans:%s:report", scanID)
48+
data, err := json.Marshal(report)
49+
50+
if err != nil {
51+
log.Debugf("Failed to serialize report to JSON: %#v", err)
52+
return err
53+
}
54+
55+
_, err = client.redis.Set(key, data, DEFAULT_EXPIRATION).Result()
56+
57+
if err != nil {
58+
log.Debugf("Failed to set report map in Redis: %#v", err)
59+
}
60+
61+
return err
62+
}
63+
64+
func (client *RedisPersistence) GetScanStatus(scanID string) string {
65+
return SCAN_STATUS_ERROR
66+
}
67+
68+
func (client *RedisPersistence) GetScanReport(scanID string) (report ScanReport, err error) {
69+
return report, err
70+
}

‎service.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ type ScanReport struct {
3131

3232
// Hash index is not addressable in Go so we use flat key-value pairing
3333
type ScanningService struct {
34-
statusMap map[string]string
35-
reportMap map[string]ScanReport
36-
scannerChannel chan ScanJob
34+
statusMap map[string]string
35+
reportMap map[string]ScanReport
36+
persistentStore RedisPersistence
37+
scannerChannel chan ScanJob
3738
}
3839

3940
type ScanJob struct {
@@ -84,7 +85,9 @@ func dockerPullImage(imageRef, user, password string) error {
8485
}
8586

8687
func (ctx *ScanningService) Init() {
87-
log.Debugf("Scanning service initialized")
88+
log.Debugf("Initalizing Scanning Service")
89+
90+
ctx.persistentStore.Init()
8891

8992
ctx.scannerChannel = make(chan ScanJob, 5)
9093
ctx.statusMap = make(map[string]string, 0)
@@ -95,6 +98,8 @@ func (ctx *ScanningService) Init() {
9598
ctx.processJob(job)
9699
}
97100
}()
101+
102+
log.Debugf("Scanning service initialized")
98103
}
99104

100105
func (ctx *ScanningService) processJob(job ScanJob) {

0 commit comments

Comments
 (0)