Fixed install_from_src failure (#798)

* Fix for issue 796

* Made install_dep modular

* fixed import

* fixed sudo commands

* Changed package name to ts_script to avoid conflict with conda script pkg"

* modified script to ts_script

Co-authored-by: Ubuntu <[email protected]>
Co-authored-by: Geeta Chauhan <[email protected]>

Author: 3 people

.circleci/config.yml

@@ -34,7 +34,7 @@ commands:
     steps:
       - run:
           name: Install build and test dependencies for torchserve
-          command: "python scripts/install_dependencies.py"
+          command: "python ts_scripts/install_dependencies.py"
 
   attach-torchserve-workspace:
     description: "Attach the torchserve directory which was saved into workspace"
@@ -59,7 +59,7 @@ commands:
     steps:
       - run:
           name: Start torchserve, Execute << parameters.collection >> API Tests, Stop torchserve
-          command: python scripts/test_api.py << parameters.collection >>
+          command: python ts_scripts/test_api.py << parameters.collection >>
           when: always
 
 
@@ -102,7 +102,7 @@ jobs:
       - install-dependencies
       - run:
           name: Execute frontend gradle tests
-          command: python scripts/test_frontend.py
+          command: python ts_scripts/test_frontend.py
       - store_artifacts:
           name: Store frontend gradle test results
           path: frontend/server/build/reports/tests/test
@@ -117,7 +117,7 @@ jobs:
       - install-dependencies
       - run:
           name: Execute python lint and unit tests
-          command: python scripts/test_torchserve.py
+          command: python ts_scripts/test_torchserve.py
       - store_artifacts:
           name: Store python Test results
           path: result_units
@@ -134,7 +134,7 @@ jobs:
       - install-torchserve
       - run:
           name: Execute sanity tests
-          command: python scripts/test_sanity.py
+          command: python ts_scripts/test_sanity.py
       - store_artifacts:
           name: Store TS logs from sanity tests
           path: logs/
@@ -171,7 +171,7 @@ jobs:
       - install-torchserve
       - run:
           name: Execute regression suite
-          command: python scripts/test_regression.py
+          command: python ts_scripts/test_regression.py
       - store_artifacts:
           name: Store server logs from regression tests
           path: test/pytest/logs/
@@ -232,7 +232,7 @@ jobs:
       - install-torchserve
       - run:
           name: Execute python lint, unit and integration tests
-          command: python scripts/test_modelarchiver.py
+          command: python ts_scripts/test_modelarchiver.py
       - store_artifacts:
           name: Store unit tests results from model archiver tests
           path: model-archiver/result_units

README.md

@@ -79,7 +79,7 @@ NOTE: This script uninstalls existing `torchserve` and `torch-model-archiver` in
 #### For Debian Based Systems/ MacOS
 
 ```
-python ./scripts/install_from_src.py
+python ./ts_scripts/install_from_src.py
 ```
 
 For information about the model archiver, see [detailed documentation](model-archiver/README.md).

binaries/build.py

@@ -6,7 +6,7 @@
 REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(REPO_ROOT)
 
-from scripts import tsutils
+from ts_scripts import tsutils
 from binaries.conda.build_packages import conda_build
 
 

binaries/install.py

@@ -6,7 +6,7 @@
 REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(REPO_ROOT)
 
-from scripts import tsutils
+from ts_scripts import tsutils
 
 
 def install():

ci/buildspec.yml

@@ -5,7 +5,7 @@ version: 0.2
 phases:
   install:
     commands:
-      - python scripts/install_dependencies.py --cuda=cu101
+      - python ts_scripts/install_dependencies.py --cuda=cu101
 
   build:
     commands:

docs/torchserve_on_win_native.md

@@ -45,7 +45,7 @@ NOTE At present, wheels for windows are not available on PyPi. However following
  - `git clone https://github.com/pytorch/serve.git`
  - `pip install click`
  - `cd serve`
- - `python .\scripts\install_from_src.py`
+ - `python .\ts_scripts\install_from_src.py`
  - Refer [Install torchserve for development](https://github.com/pytorch/serve#install-torchserve-for-development)
 
 ## Troubleshooting

docs/torchserve_on_wsl.md

@@ -47,7 +47,7 @@ cd serve
 
 ./scripts/setup_wsl_ubuntu
 export PATH=$HOME/.local/bin:$PATH
-python ./scripts/install_from_src.py
+python ./ts_scripts/install_from_src.py
 echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc
 source ~/.bashrc
 ```

test/buildspec.yml

@@ -5,7 +5,7 @@ version: 0.2
 phases:
   install:
     commands:
-      - python scripts/install_dependencies.py --cuda=cu101
+      - python ts_scripts/install_dependencies.py --cuda=cu101
 
   build:
     commands:

test/regression_tests.py

@@ -6,9 +6,9 @@
 REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(REPO_ROOT)
 
-from scripts.install_from_src import install_from_src
-from scripts.test_regression import test_regression
-from scripts.test_api import test_api
+from ts_scripts.install_from_src import install_from_src
+from ts_scripts.test_regression import test_regression
+from ts_scripts.test_api import test_api
 
 from pygit2 import Repository
 git_branch = Repository('.').head.shorthand

torchserve_sanity.py

@@ -1,10 +1,10 @@
-from scripts.test_modelarchiver import test_modelarchiver
-from scripts.test_torchserve import test_torchserve
-from scripts.install_from_src import install_from_src
-from scripts.test_sanity import test_sanity
-from scripts.shell_utils import rm_dir
-from scripts.test_frontend import test_frontend
-import scripts.tsutils as ts
+from ts_scripts.test_modelarchiver import test_modelarchiver
+from ts_scripts.test_torchserve import test_torchserve
+from ts_scripts.install_from_src import install_from_src
+from ts_scripts.test_sanity import test_sanity
+from ts_scripts.shell_utils import rm_dir
+from ts_scripts.test_frontend import test_frontend
+import ts_scripts.tsutils as ts
 import test.print_env_info as build_hdr_printer
 
 

scripts/Readme.md renamed to ts_scripts/Readme.md

@@ -5,7 +5,7 @@ The scripts are designed to execute independent of each other.
 1. [install_dependencies.py](install_dependencies.py) : Helps you install build and test dependencies for torchserve (like java, torch packages, newman, etc)
    ```
    # Example
-   python scripts/install_dependencies.py
+   python ts_scripts/install_dependencies.py
 
 2. [tsutils.py](tsutils.py.py) : Import this module in your scripts to have access to utility methods like start\stop torchserve, register\infer\unregister models, etc.
 3. [shell_utils.py](shell_utils.py) : Import this module in your scripts to have access to utility methods like download file, remove directory, unzip files, etc
@@ -17,8 +17,8 @@ The scripts are designed to execute independent of each other.
 9. [test_api.py](test_api.py) : Executes newman api collections (management\inference\increased_timeout_inference\https\all)
    ```
    # Example
-   python scripts/test_api.py management
-   python scripts/test_api.py inference
-   python scripts/test_api.py all
+   python ts_scripts/test_api.py management
+   python ts_scripts/test_api.py inference
+   python ts_scripts/test_api.py all
    ```
-10. [test_regression.py](test_regression.py) : Executes regression pytests.
+10. [test_regression.py](test_regression.py) : Executes regression pytests.

scripts/__init__.py renamed to ts_scripts/__init__.py

@@ -4,10 +4,11 @@
 
 
 class Common():
-    def __init__(self):
+    def __init__(self, sudo_cmd):
         # Assumption is nvidia-smi is installed on systems with gpu
         self.is_gpu_instance = True if os.system("nvidia-smi") == 0 else False
         self.torch_stable_url = "https://download.pytorch.org/whl/torch_stable.html"
+        self.sudo_cmd = sudo_cmd
 
     def install_java(self):
         pass
@@ -31,40 +32,42 @@ def install_python_packages(self, cuda_version):
             os.system("conda install -y conda-build")
 
     def install_node_packages(self):
-        os.system("npm install -g newman newman-reporter-html markdown-link-check")
+        os.system(f"{self.sudo_cmd}apt-get update")
+        os.system(f"{self.sudo_cmd}npm install -g newman newman-reporter-html markdown-link-check")
 
     def install_jmeter(self):
         pass
 
-    def install_ab(self):
-        pass
-
 
 class Linux(Common):
+    def __init__(self, sudo_cmd):
+        super().__init__(sudo_cmd)
+
     def install_java(self):
-        os.system("apt-get install -y openjdk-11-jdk")
+        os.system(f"{self.sudo_cmd}apt-get update")
+        os.system(f"{self.sudo_cmd}apt-get install -y openjdk-11-jdk")
 
     def install_nodejs(self):
-        os.system("curl -sL https://deb.nodesource.com/setup_14.x | bash -")
-        os.system("apt-get update")
-        os.system("apt-get install -y nodejs")
-
-    def install_ab(self):
-        os.system("apt-get install -y apache2-utils")
+        os.system(f"{self.sudo_cmd}apt-get update")
+        os.system(f"{self.sudo_cmd}curl -sL https://deb.nodesource.com/setup_14.x | {self.sudo_cmd}bash -")
+        os.system(f"{self.sudo_cmd}apt-get install -y nodejs")
 
 
 class Windows(Common):
+    def __init__(self, sudo_cmd):
+        super().__init__(sudo_cmd)
+
     def install_java(self):
         pass
 
     def install_nodejs(self):
         pass
 
-    def install_ab(self):
-        pass
-
 
 class Darwin(Common):
+    def __init__(self, sudo_cmd):
+        super().__init__(sudo_cmd)
+
     def install_java(self):
         os.system("brew tap AdoptOpenJDK/openjdk")
         os.system("brew cask install adoptopenjdk11")
@@ -76,21 +79,25 @@ def install_torch_packages(self, cuda_version=''):
         os.system(f"pip install -U -r requirements/torch.txt -f {self.torch_stable_url}")
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Install various build and test dependencies of TorchServe")
-    parser.add_argument('--cuda', default=None, choices=['cu101'], help="CUDA version for torch")
-    args = parser.parse_args()
-
+def install_dependencies(sudo_cmd='sudo ', cuda_version=None):
     os_map = {
         "Linux": Linux,
         "Windows": Windows,
         "Darwin": Darwin
     }
-    system = os_map[platform.system()]()
+    system = os_map[platform.system()](sudo_cmd)
 
     import sys
     # Sequence of installation to be maintained
     system.install_java()
     system.install_nodejs()
-    system.install_python_packages(cuda_version=args.cuda)
+    system.install_python_packages(cuda_version)
     system.install_node_packages()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Install various build and test dependencies of TorchServe")
+    parser.add_argument('--cuda', default=None, choices=['cu101'], help="CUDA version for torch")
+    args = parser.parse_args()
+
+    install_dependencies('', cuda_version=args.cuda)
+

scripts/install_dependencies.py renamed to ts_scripts/install_dependencies.py

@@ -3,11 +3,14 @@
 import time
 import shutil
 
+
 # To help discover local modules
 REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(REPO_ROOT)
 
-from scripts import tsutils as ts
+from ts_scripts import tsutils as ts
+from ts_scripts import install_dependencies as idep
+
 
 def clean_slate(): 
     print("## Uninstall existing torchserve and model archiver")
@@ -54,4 +57,5 @@ def install_from_src():
 
 
 if __name__ == '__main__':
+    idep.install_dependencies()
     install_from_src()

scripts/install_from_src.py renamed to ts_scripts/install_from_src.py

@@ -2,7 +2,7 @@
 import os
 import sys
 import argparse
-from scripts import tsutils as ts
+from ts_scripts import tsutils as ts
 
 
 REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")

scripts/install_utils renamed to ts_scripts/install_utils

@@ -3,7 +3,7 @@
 import nvgpu
 import glob
 
-from scripts import tsutils as ts
+from ts_scripts import tsutils as ts
 
 
 def run_markdown_link_checker():