Only index published PDF documents (i.e. documents not in the recycle bin). (#40)

* Only index published PDF documents.  If a document is deleted it was removed from the index and rebuilding the indexing included it again.

* Simplified conditional check for completing the PerformIndexItems method.

* Refactor to avoid breaking change in PdfValueSetValidator constructor.

* Made new method private.

---------

Co-authored-by: Andy Butland <[email protected]>

Author: justin-nevitech

src/UmbracoExamine.PDF/ConfigurePdfIndexOptions.cs

@@ -26,7 +26,7 @@ public void Configure(string name, LuceneDirectoryIndexOptions options)
             if (name.Equals(PdfIndexConstants.PdfIndexName))
             {
                 options.Analyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48);
-                options.Validator = new PdfValueSetValidator(null);
+                options.Validator = new PdfValueSetValidator(null, true);
                 options.FieldDefinitions = new FieldDefinitionCollection(
                     new FieldDefinition(PdfIndexConstants.PdfContentFieldName,FieldDefinitionTypes.FullText));
 

src/UmbracoExamine.PDF/PdfLuceneIndex.cs

@@ -1,5 +1,7 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
+using Examine;
 using Examine.Lucene;
 using Examine.Lucene.Providers;
 using Microsoft.Extensions.Logging;
@@ -31,6 +33,59 @@ public PdfLuceneIndex(ILoggerFactory loggerFactory, string name, IOptionsMonitor
 
         #endregion
 
+        /// <summary>
+        ///     Special check for invalid paths
+        /// </summary>
+        /// <param name="values"></param>
+        /// <param name="onComplete"></param>
+        protected override void PerformIndexItems(IEnumerable<ValueSet> values, Action<IndexOperationEventArgs> onComplete)
+        {
+            // We don't want to re-enumerate this list, but we need to split it into 2x enumerables: invalid and valid items.
+            // The Invalid items will be deleted, these are items that have invalid paths (i.e. moved to the recycle bin, etc...)
+            // Then we'll index the Value group all together.
+            var invalidOrValid = values.GroupBy(v =>
+            {
+                if (!v.Values.TryGetValue("path", out IReadOnlyList<object> paths) || paths.Count <= 0 || paths[0] == null)
+                {
+                    return ValueSetValidationStatus.Failed;
+                }
+
+                ValueSetValidationResult validationResult = ValueSetValidator.Validate(v);
+
+                return validationResult.Status;
+            }).ToArray();
+
+            var hasUpdates = false;
 
+            // ordering by descending so that Filtered/Failed processes first
+            foreach (IGrouping<ValueSetValidationStatus, ValueSet> group in invalidOrValid.OrderByDescending(x => x.Key))
+            {
+                switch (group.Key)
+                {
+                    case ValueSetValidationStatus.Valid:
+                        hasUpdates = true;
+
+                        //these are the valid ones, so just index them all at once
+                        base.PerformIndexItems(group.ToArray(), onComplete);
+                        break;
+                    case ValueSetValidationStatus.Failed:
+                        // don't index anything that is invalid
+                        break;
+                    case ValueSetValidationStatus.Filtered:
+                        // these are the invalid/filtered items so we'll delete them
+                        // since the path is not valid we need to delete this item in
+                        // case it exists in the index already and has now
+                        // been moved to an invalid parent.
+                        base.PerformDeleteFromIndex(group.Select(x => x.Id), null);
+                        break;
+                }
+            }
+
+            if (!hasUpdates)
+            {
+                //we need to manually call the completed method
+                onComplete(new IndexOperationEventArgs(this, 0));
+            }
+        }
     }
 }

src/UmbracoExamine.PDF/PdfValueSetValidator.cs

@@ -1,6 +1,8 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using Examine;
+using Umbraco.Cms.Core;
 using Umbraco.Cms.Infrastructure.Examine;
 using Umbraco.Extensions;
 
@@ -10,15 +12,31 @@ public class PdfValueSetValidator : ValueSetValidator
     {
         public int? ParentId { get; }
 
+        public bool PublishedValuesOnly { get; }
+
         private const string PathKey = "path";
 
-        public PdfValueSetValidator(int? parentId,
-            IEnumerable<string> includeItemTypes = null, IEnumerable<string> excludeItemTypes = null)
+        [Obsolete("Please use the constructor taking all parameters. This constructor will be removed in a future version.")]
+        public PdfValueSetValidator(
+            int? parentId,
+            IEnumerable<string> includeItemTypes = null,
+            IEnumerable<string> excludeItemTypes = null)
+            : this(parentId, false, includeItemTypes, excludeItemTypes)
+        {
+        }
+
+        public PdfValueSetValidator(
+            int? parentId,
+            bool publishedValuesOnly,
+            IEnumerable<string> includeItemTypes = null,
+            IEnumerable<string> excludeItemTypes = null)
             : base(includeItemTypes, excludeItemTypes, null, null)
         {
             ParentId = parentId;
+            PublishedValuesOnly = publishedValuesOnly;
         }
 
+        // TODO (next major): make this method private
         public bool ValidatePath(string path)
         {
             //check if this document is a descendent of the parent
@@ -33,6 +51,24 @@ public bool ValidatePath(string path)
             return true;
         }
 
+        private bool ValidateRecycleBin(string path, string category)
+        {
+            var recycleBinId = category == IndexTypes.Content
+                ? Constants.System.RecycleBinContentString
+                : Constants.System.RecycleBinMediaString;
+
+            //check for recycle bin
+            if (PublishedValuesOnly)
+            {
+                if (path.Contains(string.Concat(",", recycleBinId, ",")))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
         public override ValueSetValidationResult Validate(ValueSet valueSet)
         {
             var baseValidate = base.Validate(valueSet);
@@ -53,7 +89,7 @@ public override ValueSetValidationResult Validate(ValueSet valueSet)
 
             var filteredValues = valueSet.Values.ToDictionary(x => x.Key, x => x.Value.ToList());
 
-            bool isFiltered = !ValidatePath(path);
+            bool isFiltered = !ValidatePath(path) || !ValidateRecycleBin(path!, valueSet.Category);
 
             var filteredValueSet = new ValueSet(valueSet.Id, valueSet.Category, valueSet.ItemType, filteredValues.ToDictionary(x => x.Key, x => (IEnumerable<object>)x.Value));
             return new ValueSetValidationResult(isFiltered ? ValueSetValidationStatus.Filtered : ValueSetValidationStatus.Valid, filteredValueSet);