Add otus 18 and async scrapper

Flawless · Flawless · commit cdc0f7b2031c · 2024-01-15T22:55:07.000+04:00
diff --git a/async-web-scrapper/project.clj b/async-web-scrapper/project.clj
@@ -0,0 +1,9 @@
+(defproject web-scraper "0.1.0-SNAPSHOT"
+  :description "Simple WEB scrapper example"
+
+  :dependencies [[org.clojure/clojure "1.11.1"]
+                 [org.clojure/core.async "1.6.673"]
+                 [clj-http "3.12.3"]
+                 [org.jsoup/jsoup "1.16.1"]]
+
+  :repl-options {:init-ns async-web-scraper.core})
diff --git a/async-web-scrapper/src/async_web_scraper/core.clj b/async-web-scrapper/src/async_web_scraper/core.clj
@@ -0,0 +1,113 @@
+(ns async-web-scraper.core
+  (:require [clj-http.client :as http]
+            [clojure.core.async
+             :as async
+             :refer [<! <!! >! chan close! go pipeline pipeline-async thread]]
+            [clojure.string :as str])
+  (:import (org.jsoup Jsoup)))
+
+(defn async-request [opts]
+  (thread (http/request opts)))
+
+(defn request-page [base-url href]
+  (go (-> (<! (async-request {:url (str base-url href)
+                              :method :get}))
+          :body
+          (Jsoup/parse))))
+
+(defn extract-pagination-hrefs [page]
+  (let [external? (fn [ref] (str/includes? ref "https://"))]
+    (-> page
+        (.select "div[class^=paginator] > a[class^=link]")
+        (->> (map #(.attr % "href"))
+             (remove external?)))))
+
+(defn extract-articles-hrefs [page]
+  (let [external? (fn [ref] (str/includes? ref "https://"))]
+    (-> page
+        (.select "div[class^=card] a[class^=link]")
+        (->> (map #(.attr % "href"))
+             (remove external?)))))
+
+(defn extract-data [page]
+  (let [author (-> page
+                   (.select "div[class^=coAuthor] a p")
+                   (.text))
+        title (-> (.select page "title")
+                  .text)
+        text (-> page
+                 (.select "div[class^=articleView] p")
+                 (->> (map #(.text %))
+                      (str/join "\n")))
+        words-count (count (str/split text #"\s+"))]
+    {:title title
+     :author author
+     ;; :text text
+     :words-count words-count}))
+
+(defn async-scraping [base-url & {:keys [limit-pages entrypoint]
+                                  :or {limit-pages 5}}]
+  (let [flow-href-c (chan)
+
+        article-href-c (chan 20)
+        article-page-c (chan 20)
+
+        out-c (chan 100)]
+
+    ;; request journal pages and extract hrefs to articles from them
+    (pipeline-async 1 article-href-c
+                    (fn [href result]
+                      (go
+                        (let [flow-page (<! (request-page base-url href))
+                              articles-hrefs (extract-articles-hrefs flow-page)]
+                          (doseq [article-href articles-hrefs]
+                            (>! result article-href))
+                          (close! result))))
+                    flow-href-c)
+
+    ;; request article pages by their hrefs
+    (pipeline-async 1 article-page-c
+                    (fn [href result]
+                      (go (->> (<! (request-page base-url href))
+                               (>! result))
+                          (close! result)))
+                    article-href-c)
+
+    ;; extract data from articles pages
+    (pipeline 4 out-c (map extract-data) article-page-c)
+
+    (go (let [entry-page (<! (request-page base-url entrypoint))]
+          (doseq [href (take limit-pages (extract-pagination-hrefs entry-page))]
+            (>! flow-href-c href))
+          (close! flow-href-c)))
+
+    out-c))
+
+(comment
+  (def flow-page
+    (<!! (request-page "https://journal.tinkoff.ru"
+                       "/flows/readers-travel/")))
+
+  (def article-page
+    (<!! (request-page "https://journal.tinkoff.ru"
+                       "/roadtrip-usa-hidden-cost/")))
+
+  (extract-pagination-hrefs flow-page)
+  (extract-articles-hrefs flow-page)
+  ;; => ("/13-days-in-turkey/"
+  ;;     "/travel-with-kids-to-krasnoyarsk/"
+  ;;     ...
+  ;;     "/roadtrip-usa-hidden-cost/")
+
+  (extract-data article-page)
+
+  (def ch (async-scraping "https://journal.tinkoff.ru"
+                          :entrypoint "/flows/readers-travel/"
+                          :limit-pages 5))
+
+  (<!! ch)
+
+  (count (<!! (async/into [] ch)))
+
+  ;; => 114
+  )
diff --git a/async-web-scrapper/test/web_scraper/core_test.clj b/async-web-scrapper/test/web_scraper/core_test.clj
@@ -0,0 +1,4 @@
+(ns web-scraper.core-test
+  (:require [clojure.test :refer :all]
+            [web-scrapper.core :refer :all]))
+
diff --git a/otus-18/README.md b/otus-18/README.md
@@ -0,0 +1,16 @@
+### Урок #18
+
+core.async
+
+https://github.com/clojure/core.async
+
+API Docs
+
+https://clojure.github.io/core.async/
+
+### ДЗ
+#### Научиться работать с https://pokeapi.co/ в асинхронном стиле.
+
+- Научиться получать список покемонов — это отправная точка для сбора информации.
+- Получить для каждого покемона, упомянутого в ответе на первый запрос, его имя, наименования типов на заданном языке (их может быть несколько). Эти данные можно получить асинхронно, но результата следует подождать и сохранить оный в подходящие структуры данных.
+- Типы покемонов — справочные данные. Следует озаботиться получением полного перечня типов с их наименованиями, чтобы затем только обращаться к сохранённому справочнику при добавлении наименований типов к данным покемона.
diff --git a/otus-18/project.clj b/otus-18/project.clj
@@ -0,0 +1,9 @@
+(defproject otus-18 "0.1.0-SNAPSHOT"
+  :description "FIXME: write description"
+  :url "http://example.com/FIXME"
+
+  :dependencies [[org.clojure/clojure "1.11.1"]
+                 [org.clojure/core.async "1.6.673"]
+                 [clj-http "3.12.3"]
+                 [clj-http-fake "1.0.4"]
+                 [cheshire "5.11.0"]])
diff --git a/otus-18/src/otus_18/async.clj b/otus-18/src/otus_18/async.clj
@@ -0,0 +1,110 @@
+(ns otus-18.async
+  (:require [clojure.core.async :as a :refer [<! <!! >! >!!
+                                              alts! chan close!
+                                              go go-loop
+                                              pipeline-async pipeline
+                                              thread timeout]]))
+
+;; Всем ли хорошо видно?
+
+;; * Каналы (Channels)
+
+(def echo-chan (chan))
+(go (println (<! echo-chan)))
+(>!! echo-chan "bottle")
+; => true
+; => bottle
+
+;; * Буферизация (Buffers)
+
+(def echo-buffer (chan 2))
+(>!! echo-buffer :hello)
+; => true
+(>!! echo-buffer :world)
+; => true
+(>!! echo-buffer :!)
+
+; Третий вызов будет блокирующим, т.к. буфер переполнен
+
+;; Note: `sliding-buffer` - при переполнении старые данные будут удаляться из буфера
+;;       `dropping-buffer` - новые данные не будут попадать в переполненный буфер
+;;       при их использовании `>!!` никогда не заблокирует поток выполнения
+
+;; * Блокирование и Паркинг (Blocking and Parking)
+
+;; `>!`, `<!` - могут быть использованы только внутри `go`-блока
+;; `>!!`, `<!!` - могут быть использованы как в `go`-блоках, так и в основном коде
+
+(def n-ch (chan))
+(doseq [n (range 1000)]
+  (go (>! n-ch n)))
+
+;; `>!`, `<!` - park thread
+;; `>!!`, `<!!` - block thread
+
+;; * thread
+
+(def ch (chan))
+
+(time (do
+        (doseq [n (range 9)]
+          (go
+            (Thread/sleep 1000)
+            (>! ch n)))
+        (doseq [_ (range 9)]
+          (<!! ch))))
+
+; => "Elapsed time: 2016.62955 msecs"
+
+(def another-echo-chan (chan))
+(thread (println (<!! another-echo-chan)))
+(>!! another-echo-chan "hi")
+; => true
+; => hi
+
+;; * alts!
+
+(def ch-a (chan))
+(def ch-b (chan))
+(go (println (alts! [ch-a ch-b])))
+(>!! ch-b "bottle")
+
+(go (println (alts! [ch-a ch-b (timeout 1000)])))
+
+;; * pipeline
+
+
+(def capitalizer (map clojure.string/capitalize))
+
+(def input (chan))
+(def output (chan))
+
+(go-loop []
+  (when-let [x (<! output)]
+    (println x)
+    (recur)))
+
+(pipeline 1 output capitalizer input)
+
+(>!! input "hello")
+
+(close! input)
+
+(<!! output)
+
+;; * pipeline-async
+
+(def ca> (chan 1))
+(def cb> (chan 1))
+
+(defn c-af [val result] ; notice the signature is different for `pipeline-async`, it includes a channel
+  (go (<! (timeout 1000))
+      (>! result (str val "!!!"))
+      (>! result (str val "!!!"))
+      (>! result (str val "!!!"))
+      (close! result)))
+
+(pipeline-async 1 cb> c-af ca>)
+
+(go (println (<! cb>)))
+(>!! ca> "hello")
diff --git a/otus-18/src/otus_18/homework/pokemons.clj b/otus-18/src/otus_18/homework/pokemons.clj
@@ -0,0 +1,19 @@
+(ns otus-18.homework.pokemons)
+
+(def base-url     "https://pokeapi.co/api/v2")
+(def pokemons-url (str base-url "/pokemon"))
+(def type-path    (str base-url "/type"))
+
+(defn extract-pokemon-name [pokemon]
+  (:name pokemon))
+
+(defn extract-type-name [pokemon-type lang]
+  (->> (:names pokemon-type)
+       (filter (fn [type-name] (= lang (-> type-name :language :name))))
+       (first)
+       :name))
+
+(defn get-pokemons
+  "Асинхронно запрашивает список покемонов и название типов в заданном языке. Возвращает map, где ключами являются
+  имена покемонов (на английском английский), а значения - коллекция названий типов на заданном языке."
+  [& {:keys [limit lang] :or {limit 50 lang "ja"}}])
diff --git a/otus-18/test/otus_18/homework/pokemons_test.clj b/otus-18/test/otus_18/homework/pokemons_test.clj
@@ -0,0 +1,48 @@
+(ns otus-18.homework.pokemons-test
+  (:require
+   [cheshire.core :as cheshire]
+   [otus-18.homework.pokemons :as subject]
+   [clj-http.fake :refer [with-global-fake-routes-in-isolation]]
+   [clojure.test :refer [deftest is]]))
+
+(deftest get-pokemons-test
+  (with-global-fake-routes-in-isolation
+    {{:address "https://pokeapi.co/api/v2/pokemon" :query-params {:limit 50 :lang "ja"}}
+     (fn [_request]
+       {:status 200
+        :body (cheshire/generate-string
+               {:results
+                [{:name "pikachu"
+                  :url "https://pokeapi.co/api/v2/pokemon/1/"}]})})
+     {:address "https://pokeapi.co/api/v2/pokemon/pikachu" :query-params {:lang "ja"}}
+     (fn [_request]
+       {:status 200
+        :body (cheshire/generate-string
+               {:name "pikachu"
+                :types [{:slot 1
+                         :type {:name "electric",
+                                :url "https://pokeapi.co/api/v2/type/13/"}}]})})
+    {:address "https://pokeapi.co/api/v2/pokemon/1/" :query-params {:lang "ja"}}
+     (fn [_request]
+       {:status 200
+        :body (cheshire/generate-string
+               {:name "pikachu"
+                :types [{:slot 1
+                         :type {:name "electric",
+                                :url "https://pokeapi.co/api/v2/type/13/"}}]})})
+     "https://pokeapi.co/api/v2/type/electric"
+     (fn [_request]
+       {:status 200
+        :body (cheshire/generate-string
+               {:names [{:language {:name "ja",
+                                    :url  "https://pokeapi.co/api/v2/language/11/"},
+                         :name     "でんき"}]})})
+    "https://pokeapi.co/api/v2/type/13/"
+    (fn [_request]
+      {:status 200
+       :body (cheshire/generate-string
+              {:names {:language {:name "ja",
+                                  :url "https://pokeapi.co/api/v2/language/11/"}}})})}
+
+    (is (= {"pikachu" ["でんき"]}
+           (subject/get-pokemons :lang "ja")))))
