diff --git a/library/src/uk/co/senab/bitmapcache/BitmapMemoryLruCache.java b/library/src/uk/co/senab/bitmapcache/BitmapMemoryLruCache.java
index 70ae13d..dff29c6 100644
--- a/library/src/uk/co/senab/bitmapcache/BitmapMemoryLruCache.java
+++ b/library/src/uk/co/senab/bitmapcache/BitmapMemoryLruCache.java
@@ -83,7 +83,7 @@ Bitmap getBitmapFromRemoved(final int width, final int height) {
             while (it.hasNext()) {
                 CacheableBitmapDrawable value = it.next().get();
 
-                if (value != null && value.isBitmapValid() && value.isBitmapMutable()) {
+                if (value != null && value.isBitmapValid() && value.isBitmapMutable() && !value.isBeingDisplayed() && !value.isReferencedByCache()) {
                     if (value.getIntrinsicWidth() == width
                             && value.getIntrinsicHeight() == height) {
                         it.remove();