initializing repo with working copy

Author: digitaldreams

.gitignore

@@ -0,0 +1,2 @@
+/storage
+/.idea

Download.py

@@ -0,0 +1,52 @@
+import urllib.parse
+import urllib.request
+import os.path
+import errno
+import SaveFile
+
+
+class Download():
+    def __init__(self, file_name, path=''):
+        self.links = set()
+        self.completed = set()
+        self.file_name = file_name
+        self.file_to_set()
+        self.path = path
+
+    """
+    Load links from file and set to Set()
+    """
+
+    def file_to_set(self):
+        if not os.path.exists(self.file_name):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.file_name)
+        with open(self.file_name, 'rt') as f:
+            for line in f:
+                self.links.add(line.replace('\n', ''))
+        return sorted(self.links)
+
+    """
+    Start of the downloading
+    """
+
+    def start(self):
+        for file in self.links:
+            try:
+                img = SaveFile.SaveFile(file, self.path)
+                img.save()
+            except:
+                continue
+            self.completed.add(file)
+        self.set_to_file()
+
+    """
+    Update links txt file
+    """
+
+    def set_to_file(self):
+        remaining = self.links.difference(self.completed)
+        with open(self.file_name, 'w') as f:
+            if len(remaining) > 0:
+                for line in self.links:
+                    f.write(line + "\n")
+            f.write("")

ImgFinder.py

@@ -0,0 +1,42 @@
+from html.parser import HTMLParser
+import urllib.parse
+import functions
+
+
+class ImgFinder(HTMLParser):
+    def __init__(self, page_url):
+        urlres = urllib.parse.urlparse(page_url)
+        self.page_url = page_url
+        self.base_url = urlres.netloc
+        self.folder = functions.get_folder_name(urlres.netloc)
+        self.path = urlres.path
+        self.src = set()
+        HTMLParser.__init__(self)
+    """
+    This function called by HTMLParser internally. We modify it to make our work
+    """
+    def handle_starttag(self, tag, attrs):
+        if tag == 'img':
+            for (attr, value) in attrs:
+                if attr == 'src':
+                    fullUrl = urllib.parse.urljoin(self.base_url, value)
+                    self.src.add(fullUrl)
+                else:
+                    continue
+
+    def getSrc(self):
+        return self.src
+
+    def get_base_url(self):
+        return self.base_url
+
+    def save_to_file(self):
+        file_name = self.folder_path()+ self.path + '.txt'
+        with open(file_name, 'w') as f:
+            for line in sorted(self.src):
+                f.write(line + '\n')
+
+        return file_name
+
+    def folder_path(self):
+        return "storage/" + self.folder

SaveFile.py

@@ -0,0 +1,20 @@
+import urllib.request
+import os.path
+
+
+class SaveFile:
+    def __init__(self, file_name, path=''):
+        self.file_name = file_name
+
+        self.base_name = os.path.basename(urllib.request.urlparse(file_name).path)
+        self.path = path
+
+    def save(self):
+        try:
+            if len(self.path) > 0:
+                full_file_path = self.path + '/' + self.base_name
+            else:
+                full_file_path = self.base_name
+            urllib.request.urlretrieve(self.file_name, full_file_path)
+        except Exception as e:
+            print(str(e))

__pycache__/Download.cpython-36.pyc

@@ -0,0 +1,43 @@
+import urllib.request
+import ImgFinder
+import urllib.parse
+import os.path
+
+
+def gather_img_src(page_url):
+    try:
+        html = html_string(page_url)
+        finder = ImgFinder.ImgFinder(page_url)
+        finder.feed(html)
+    except Exception as e:
+        print(str(e))
+        return set()
+
+    return finder.getSrc()
+
+
+def create_project_folder(page_url):
+    base_url = get_folder_name(urllib.parse.urlparse(page_url).netloc)
+    if not os.path.exists("storage/" + base_url):
+        os.makedirs("storage/" + base_url)
+
+
+def html_string(page_url):
+    html_string = ''
+    try:
+        response = urllib.request.urlopen(page_url)
+        if 'text/html' in response.getheader('Content-Type'):
+            html_bytes = response.read()
+            html_string = html_bytes.decode("utf-8")
+
+    except Exception as e:
+        print(str(e))
+    return html_string
+
+
+def get_folder_name(base_url):
+    parts = base_url.split(".")
+    if len(parts) == 3:
+        return parts[1]
+    else:
+        return parts.join("-")

__pycache__/ImgFinder.cpython-36.pyc

@@ -0,0 +1,15 @@
+import urllib.request
+import urllib.parse
+from functions import *
+import ImgFinder
+import Download
+
+if __name__ == '__main__':
+    PAGE_URL = 'http://www.fdfashionbd.com/gallarey'
+    create_project_folder(PAGE_URL)
+    finder = ImgFinder.ImgFinder(PAGE_URL)
+    finder.feed(html_string(PAGE_URL))
+    file_name = finder.save_to_file()
+    # start downloading images
+    down = Download.Download(file_name, finder.folder_path())
+    down.start()

__pycache__/SaveFile.cpython-36.pyc

@@ -0,0 +1,5 @@
+import SaveFile
+
+if __name__=='__main__':
+    file=SaveFile.SaveFile('https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png',"storage")
+    file.save()