diff --git a/pkg/v1/remote/referrers.go b/pkg/v1/remote/referrers.go
index e30ca57ed..48e3835f9 100644
--- a/pkg/v1/remote/referrers.go
+++ b/pkg/v1/remote/referrers.go
@@ -66,7 +66,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string,
 	}
 
 	var b []byte
-	if resp.StatusCode == http.StatusOK {
+	if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Type") == string(types.OCIImageIndex) {
 		b, err = io.ReadAll(resp.Body)
 		if err != nil {
 			return nil, err