Some small parser improvements

Mitch528 · Mar 25, 2016 · 91cacb6 · 91cacb6
1 parent ee271e3
commit 91cacb6
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 46 deletions.
diff --git a/WebNovelConverter/Extensions/ElementExtensions.cs b/WebNovelConverter/Extensions/ElementExtensions.cs
@@ -19,9 +19,13 @@ public static IElement FirstWhereHasClass(
                     where e.HasAttribute("class")
                     where filter?.Invoke(e) ?? true
                     let names = e.GetAttribute("class").Split(' ')
-                    from name in classes
-                    where names.Any(p => p.Equals(name, StringComparison.OrdinalIgnoreCase))
-                    orderby classes.IndexOf(name)
+                    let a = (from cl in classes
+                             from name in names
+                             where cl.Equals(name, StringComparison.OrdinalIgnoreCase)
+                             select cl).FirstOrDefault()
+                    where a != null
+                    let index = classes.IndexOf(a)
+                    orderby index
                     select e).FirstOrDefault();
         }
 

diff --git a/WebNovelConverter/MainForm.cs b/WebNovelConverter/MainForm.cs
@@ -38,7 +38,6 @@ private void MainForm_Load(object sender, EventArgs e)
             _sources.Add(new RoyalRoadLSource());
             _sources.Add(new BakaTsukiSource());
             _sources.Add(new BlogspotSource());
-            _sources.Add(new WuxiaWorldSource());
             _sources.Add(new NovelsNaoSource());
 
             websiteTypeComboBox.SelectedIndex = 0;

diff --git a/WebNovelConverter/Sources/WordPressSource.cs b/WebNovelConverter/Sources/WordPressSource.cs
@@ -53,7 +53,8 @@ public class WordPressSource : WebNovelSource
         {
             "entry-title",
             "post-title",
-            "page-title"
+            "page-title",
+            "title-block"
         };
 
         protected readonly List<string> NextChapterNames = new List<string>
@@ -63,6 +64,17 @@ public class WordPressSource : WebNovelSource
             "Next"
         };
 
+
+        protected readonly List<string> NavigationNames = new List<string>
+        {
+            "Next Chapter",
+            "Next",
+            "Previous Chapter",
+            "Prev",
+            "Table of Contents",
+            "Index"
+        };
+
         public WordPressSource() : base("WordPress")
         {
         }
@@ -123,8 +135,8 @@ public override async Task<WebNovelChapter> GetChapterAsync(ChapterLink link,
 
         protected virtual WebNovelChapter ParseChapter(IElement rootElement, CancellationToken token = default(CancellationToken))
         {
-            IElement element = rootElement.FirstWhereHasClass(PostClasses)
-                                    ?? rootElement.Descendents<IElement>().FirstOrDefault(p => p.LocalName == "article");
+            IElement articleElement = rootElement.Descendents<IElement>().FirstOrDefault(p => p.LocalName == "article");
+            IElement element = rootElement.FirstWhereHasClass(PostClasses) ?? articleElement;
 
             if (element != null)
                 RemoveBloat(element);
@@ -148,12 +160,14 @@ public override async Task<WebNovelChapter> GetChapterAsync(ChapterLink link,
                     chapterNameElement = chNameLinkElement;
             }
 
-            IElement nextChapterElement = (from e in rootElement.Descendents<IElement>() ?? new List<IElement>()
+            IElement nextChapterElement = (from e in articleElement?.Descendents<IElement>() ?? rootElement.Descendents<IElement>()
                                            where e.LocalName == "a"
                                            let text = e.Text()
                                            let a = NextChapterNames.FirstOrDefault(p => text.IndexOf(p, StringComparison.OrdinalIgnoreCase) >= 0)
                                            where a != null || (e.HasAttribute("rel") && e.GetAttribute("rel") == "next")
-                                           orderby NextChapterNames.IndexOf(a)
+                                           let index = NextChapterNames.IndexOf(a)
+                                           let o = index >= 0 ? index : int.MaxValue
+                                           orderby o
                                            select e).FirstOrDefault();
 
             WebNovelChapter chapter = new WebNovelChapter();
@@ -165,6 +179,7 @@ orderby NextChapterNames.IndexOf(a)
             if (element != null)
             {
                 RemoveNavigation(element);
+                RemoveScriptStyleElements(element);
 
                 chapter.ChapterName = chapterNameElement?.Text()?.Trim();
                 chapter.Content = element.InnerHtml;
@@ -207,6 +222,28 @@ protected virtual void RemoveBloat(IElement element)
 
         protected virtual void RemoveNavigation(IElement element)
         {
+            var navElements = from e in element.Descendents<IElement>()
+                              where e.LocalName == "a"
+                              let text = e.Text()
+                              where NavigationNames.Any(p => text.IndexOf(p, StringComparison.OrdinalIgnoreCase) >= 0)
+                              select e;
+
+            foreach (IElement e in navElements.ToList())
+            {
+                e.Remove();
+            }
+        }
+
+        protected virtual void RemoveScriptStyleElements(IElement element)
+        {
+            var elements = from e in element.Descendents<IElement>()
+                           where e.LocalName == "script" || e.LocalName == "style"
+                           select e;
+
+            foreach (IElement e in elements.ToList())
+            {
+                e.Remove();
+            }
         }
     }
 }
diff --git a/WebNovelConverter/Sources/WuxiaWorldSource.cs b/WebNovelConverter/Sources/WuxiaWorldSource.cs