diff --git a/encryption.go b/encryption.go
index f5142cc..328a32f 100644
--- a/encryption.go
+++ b/encryption.go
@@ -33,9 +33,9 @@ import (
 	"github.com/containers/ocicrypt/keywrap/pkcs11"
 	"github.com/containers/ocicrypt/keywrap/pkcs7"
 	"github.com/opencontainers/go-digest"
-	log "github.com/sirupsen/logrus"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
 // EncryptLayerFinalizer is a finalizer run to return the annotations to set for
@@ -143,6 +143,9 @@ func EncryptLayer(ec *config.EncryptConfig, encOrPlainLayerReader io.Reader, des
 
 		newAnnotations := make(map[string]string)
 		keysWrapped := false
+		if len(keyWrapperAnnotations) == 0 {
+			return nil, errors.New("missing Annotations needed for decryption")
+		}
 		for annotationsID, scheme := range keyWrapperAnnotations {
 			b64Annotations := desc.Annotations[annotationsID]
 			keywrapper := GetKeyWrapper(scheme)
@@ -211,6 +214,9 @@ func DecryptLayer(dc *config.DecryptConfig, encLayerReader io.Reader, desc ocisp
 func decryptLayerKeyOptsData(dc *config.DecryptConfig, desc ocispec.Descriptor) ([]byte, error) {
 	privKeyGiven := false
 	errs := ""
+	if len(keyWrapperAnnotations) == 0 {
+		return nil, errors.New("missing Annotations needed for decryption")
+	}
 	for annotationsID, scheme := range keyWrapperAnnotations {
 		b64Annotation := desc.Annotations[annotationsID]
 		if b64Annotation != "" {