Merge pull request #108 from fosslight/develop

Be availble to download src from rubygem

src/fosslight_util/download.py

@@ -22,6 +22,7 @@
 import time
 import threading
 import platform
+import subprocess
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -64,6 +65,8 @@ def parse_src_link(src_link):
         if src_link.startswith("git://github.com/"):
             src_link_changed = change_src_link_to_https(src_link_split[0])
         else:
+            if "rubygems.org" in src_link:
+                src_info["rubygems"] = True
             src_link_changed = src_link_split[0]
 
         branch_info = [s for s in src_link_split if s.startswith('branch')]
@@ -100,7 +103,7 @@ def main():
     if args.log_dir:
         log_dir = args.log_dir
 
-    if src_link == "":
+    if not src_link:
         print_help_msg_download()
     else:
         cli_download_and_extract(src_link, target_dir, log_dir)
@@ -114,9 +117,10 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
     log_file_name = "fosslight_download_" + \
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
+    link = link.strip()
 
     try:
-        if link == "":
+        if not link:
             success = False
             msg = "Need a link to download."
         elif os.path.isfile(target_dir):
@@ -127,14 +131,19 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
             link = src_info.get("url", "")
             tag = ''.join(src_info.get("tag",  "")).split('=')[-1]
             branch = ''.join(src_info.get("branch", "")).split('=')[-1]
+            is_rubygems = src_info.get("rubygems", False)
 
-            if not download_git_clone(link, target_dir, checkout_to, tag, branch):
+            # General download (git clone, wget)
+            if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
                 success, downloaded_file = download_wget(link, target_dir, compressed_only)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True)
+            # Download from rubygems.org
+            elif is_rubygems and shutil.which("gem"):
+                success = gem_download(link, target_dir, checkout_to)
     except Exception as error:
         success = False
         msg = str(error)
@@ -164,10 +173,10 @@ def get_ref_to_checkout(checkout_to, ref_list):
 
 
 def decide_checkout(checkout_to="", tag="", branch=""):
-    if checkout_to != "":
+    if checkout_to:
         ref_to_checkout = checkout_to
     else:
-        if branch != "":
+        if branch:
             ref_to_checkout = branch
         else:
             ref_to_checkout = tag
@@ -219,7 +228,7 @@ def download_wget(link, target_dir, compressed_only):
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
         ret, new_link = get_downloadable_url(link)
-        if ret and new_link != "":
+        if ret and new_link:
             link = new_link
 
         if compressed_only:
@@ -240,7 +249,7 @@ def download_wget(link, target_dir, compressed_only):
         else:
             del alarm
 
-        if downloaded_file != "":
+        if downloaded_file:
             success = True
             logger.debug(f"wget - downloaded: {downloaded_file}")
     except Exception as error:
@@ -263,6 +272,7 @@ def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
 
 
 def extract_compressed_file(fname, extract_path, remove_after_extract=True):
+    success = True
     try:
         is_compressed_file = True
         if os.path.isfile(fname):
@@ -285,6 +295,7 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True):
                 decompress_bz2(fname, extract_path)
             else:
                 is_compressed_file = False
+                success = False
                 logger.warning(f"Unsupported file extension: {fname}")
 
             if remove_after_extract and is_compressed_file:
@@ -294,8 +305,8 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True):
             logger.warning(f"Not a file: {fname}")
     except Exception as error:
         logger.error(f"Extract - failed: {error}")
-        return False
-    return True
+        success = False
+    return success
 
 
 def decompress_bz2(source_file, dest_path):
@@ -322,5 +333,47 @@ def unzip(source_file, dest_path):
     return True
 
 
+def get_gem_version(checkout_to, ver_in_url=""):
+    gem_ver = ""
+    ver_found = False
+    if checkout_to:
+        ver_found = True
+        gem_ver = checkout_to
+    elif ver_in_url:
+        ver_found = True
+        gem_ver = ver_in_url
+    return gem_ver, ver_found
+
+
+def gem_download(link, target_dir, checkout_to):
+    ver_in_url = ""
+    success = True
+    try:
+        # EX) https://rubygems.org/gems/json/versions/2.6.2 -> get 'json'
+        gem_name = link.split("rubygems.org/gems/")[-1].split('/')[0]
+        # Ex) if version info. is included in url, json/versions/2.6.2 -> get '2.6.2'
+        if 'versions/' in link:
+            ver_in_url = link.split('versions/')[-1]
+        gem_ver, ver_found = get_gem_version(checkout_to, ver_in_url)
+
+        # gem fetch
+        if ver_found:
+            fetch_cmd = ['gem', 'fetch', gem_name, '-v', gem_ver]
+        else:
+            fetch_cmd = ['gem', 'fetch', gem_name]
+        fetch_result = subprocess.check_output(fetch_cmd, universal_newlines=True)
+        fetch_result = fetch_result.replace('\n', '').split(' ')[-1]
+        downloaded_gem = f"{fetch_result}.gem"
+
+        # gem unpack
+        subprocess.check_output(['gem', 'unpack', downloaded_gem], universal_newlines=True)
+        # move unpacked file to target directory
+        shutil.move(fetch_result, target_dir)
+    except Exception as error:
+        success = False
+        logger.warning(f"gem download - failed: {error}")
+    return success
+
+
 if __name__ == '__main__':
     main()