Fix docker health state

Author: thielepaul

container/docker/handler.go

@@ -68,9 +68,8 @@ type dockerContainerHandler struct {
 	creationTime time.Time
 
 	// Metadata associated with the container.
-	envs         map[string]string
-	labels       map[string]string
-	healthStatus string
+	envs   map[string]string
+	labels map[string]string
 
 	// Image name used for this container.
 	image string
@@ -93,6 +92,9 @@ type dockerContainerHandler struct {
 	reference info.ContainerReference
 
 	libcontainerHandler *containerlibcontainer.Handler
+
+	// the docker client is needed to inspect the container and get the health status
+	client docker.APIClient
 }
 
 var _ container.ContainerHandler = &dockerContainerHandler{}
@@ -201,10 +203,7 @@ func newDockerContainerHandler(
 		labels:             ctnr.Config.Labels,
 		includedMetrics:    metrics,
 		zfsParent:          zfsParent,
-	}
-	// Health status may be nil if no health check is configured
-	if ctnr.State.Health != nil {
-		handler.healthStatus = ctnr.State.Health.Status
+		client:             client,
 	}
 	// Timestamp returned by Docker is in time.RFC3339Nano format.
 	handler.creationTime, err = time.Parse(time.RFC3339Nano, ctnr.Created)
@@ -328,13 +327,20 @@ func (h *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) {
 // TODO(vmarmol): Get from libcontainer API instead of cgroup manager when we don't have to support older Dockers.
 func (h *dockerContainerHandler) GetStats() (*info.ContainerStats, error) {
 	stats, err := h.libcontainerHandler.GetStats()
-
-	stats.Health.Status = h.healthStatus
-
 	if err != nil {
 		return stats, err
 	}
 
+	// We assume that if Inspect fails then the container is not known to docker.
+	ctnr, err := h.client.ContainerInspect(context.Background(), h.reference.Id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inspect container %q: %v", h.reference.Id, err)
+	}
+
+	if ctnr.State.Health != nil {
+		stats.Health.Status = ctnr.State.Health.Status
+	}
+
 	// Get filesystem stats.
 	err = FsStats(stats, h.machineInfoFactory, h.includedMetrics, h.storageDriver,
 		h.fsHandler, h.fsInfo, h.poolName, h.rootfsStorageDir, h.zfsParent)

metrics/prometheus.go

@@ -143,9 +143,12 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 					return metricValues{{
 						// inline if to check if s.health.status = healthy
 						value: func(s *info.ContainerStats) float64 {
-							if s.Health.Status == "healthy" {
+							switch s.Health.Status {
+							case "healthy":
 								return 1
-							} else {
+							case "": // if container has no health check defined
+								return -1
+							default: // starting or unhealthy
 								return 0
 							}
 						}(s),