Ron/platform wheels support (microsoft#161)

* Merge release 1.0.0a21 (microsoft#155)

* Use SqlToolsService built on .NET Core 2.0 and a build script updates (microsoft#131)

* Bump version to 1.0.0a19

* Use .NET Core 2.0 RTM built sqltoolsservice

* Add build script to upload to azure blob storage

* Upgrade to VS 2017

* Remove 3.3 as supported Python version

* Fix perf issue where main event loop takes 100% of CPU (microsoft#132)

Fix perf issue where main event loop takes 100% of CPU

We have a 2 threads:
Thread microsoft#1 runs in a loop polling the response queue
Thread microsoft#2 runs in a loop decoding responses from the sqltoolsservice over stdout and posting them to the response queue

Since thread microsoft#1 doesn't sleep, it's takes 100% CPU. In addition, running python 2.7 on windows, microsoft#2 doesn’t preempt the CPU due to microsoft#1 taking all of the CPU cycles, so no response is processed.

Fix is simple – thread microsoft#1 needs to sleep so thread microsoft#2 can get scheduled and get it’s work done.

* Refine event loop perf fix in main.py

Refine event loop perf fix in main.py

* Fixing regular expression

Previous regex would result in release:a1 and release_version: 12.
Modified the regex for part Release to only pick up lower case letters.

* Adding missing forward slash on test pypi url

* fixing typos/grammar (microsoft#138)

fixing typos/grammar.

* Updating to release version 1.0.0a20.

* Create doc for official msft docs page

* Updated documentation page with usage_guide

* Added link to download adventureworks

* Updated with sqlcmd usage

* Added in run and cloud shell support

* Update/consolidate linux install (microsoft#153)

* universal linux wheel gen and setup update.

* Updating version cfg.

* Updating sqltoolsservice container.

* Updating spacing for flake8.

* Updating team email. (microsoft#154)

* Updating mssqltoolsservice to be integrated as a package of mssqlscripter.

* Updating sqltoolsservice to be loaded from the repro instead of storage account.

* Fix index file generation for daily storage account.

* Fixing manylinux1 tag.

* Updating platform tag for win x64.

* Renaming sqltoolsservice win x64 folder.

* Adding platform tags for win_amd64, manylinux1_x86_64, manylinux1_i686.

* version bumping to 1.0.0a22.

* Flake8 format fixes.

* Erroring out when build receives invalid flag.

Author: MrMeemus

.bumpversion.cfg

@@ -1,6 +1,5 @@
 [bumpversion]
-current_version = 1.0.0a21
-
+current_version = 1.0.0a22
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+))(?P<release_version>\d+)
 serialize = 
 	{major}.{minor}.{patch}{release}{release_version}
@@ -14,9 +13,5 @@ values =
 
 [bumpversion:file:setup.py]
 
-[bumpversion:file:mssqltoolsservice/setup.py]
-
 [bumpversion:file:mssqlscripter/__init__.py]
 
-[bumpversion:file:mssqltoolsservice/mssqltoolsservice/__init__.py]
-

.gitignore

@@ -93,7 +93,7 @@ ENV/
 
 # sqltoolsservice binaries
 /mssqlscripter/sqltoolsservice/*
-/mssqltoolsservice/mssqltoolsservice/bin/*
+/mssqlscripter/mssqltoolsservice/bin/*
 
 
 # VSCode configuration

MANIFEST.in

@@ -1,2 +1,4 @@
 include LICENSE.txt
 include README.rst
+include mssqlscripter/mssqltoolsservice/bin/*
+include mssqlscripter/mssqltoolsservice/bin/*/*

build.py

@@ -6,23 +6,53 @@
 # --------------------------------------------------------------------------------------------
 
 from __future__ import print_function
+from azure.storage.blob import BlockBlobService, ContentSettings
 import os
-import re
 import sys
-import tempfile
 import utility
-from azure.storage.blob import BlockBlobService, ContentSettings
+import mssqlscripter.mssqltoolsservice.external as mssqltoolsservice
 
 AZURE_STORAGE_CONNECTION_STRING = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
 BLOB_CONTAINER_NAME = 'simple'
-UPLOADED_PACKAGE_LINKS = [] 
+UPLOADED_PACKAGE_LINKS = []
 
 
 def print_heading(heading, f=None):
     print('{0}\n{1}\n{0}'.format('=' * len(heading), heading), file=f)
 
 
-def upload_index_file(service, blob_name, title, links):
+def build(platform_names):
+    """
+        Builds mssql-scripter package.
+    """
+    print_heading('Cleanup')
+
+    # clean
+    utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
+    utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
+
+    print_heading('Running setup')
+
+    # install general requirements.
+    utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
+
+    # convert windows line endings to unix for mssql-cli bash script
+    utility.exec_command('python dos2unix.py mssql-scripter mssql-scripter', utility.ROOT_DIR)
+
+    for platform in platform_names:
+        mssqltoolsservice.copy_sqltoolsservice(platform)
+
+        print_heading('Building mssql-scripter {} wheel package package'.format(platform))
+        utility.exec_command('python --version', utility.ROOT_DIR)
+        utility.exec_command(
+            'python setup.py check -r -s bdist_wheel --plat-name {}'.format(platform), 
+            utility.ROOT_DIR,
+            continue_on_error=False)
+        
+        mssqltoolsservice.clean_up_sqltoolsservice()
+
+
+def _upload_index_file(service, blob_name, title, links):
     print('Uploading index file {}'.format(blob_name))
     service.create_blob_from_text(
         container_name=BLOB_CONTAINER_NAME,
@@ -37,23 +67,23 @@ def upload_index_file(service, blob_name, title, links):
             content_language=None,
             content_md5=None,
             cache_control=None
-        )
+            )
     )
 
 
-def gen_pkg_index_html(service, pkg_name):
+def _gen_pkg_index_html(service, pkg_name):
     links = []
     index_file_name = pkg_name+'/'
     for blob in list(service.list_blobs(BLOB_CONTAINER_NAME, prefix=index_file_name)):
         if blob.name == index_file_name:
             # Exclude the index file from being added to the list
             continue
         links.append(blob.name.replace(index_file_name, ''))
-    upload_index_file(service, index_file_name, 'Links for {}'.format(pkg_name), links)
+    _upload_index_file(service, index_file_name, 'Links for {}'.format(pkg_name), links)
     UPLOADED_PACKAGE_LINKS.append(index_file_name)
 
 
-def upload_package(service, file_path, pkg_name):
+def _upload_package(service, file_path, pkg_name):
     print('Uploading {}'.format(file_path))
     file_name = os.path.basename(file_path)
     blob_name = '{}/{}'.format(pkg_name, file_name)
@@ -62,63 +92,84 @@ def upload_package(service, file_path, pkg_name):
         blob_name=blob_name,
         file_path=file_path
     )
-    gen_pkg_index_html(service, pkg_name)
-
 
-def build(options):
 
-    supported_actions = ['nightly']
-    action = None
+def validate_package(platform_names):
+    """
+        Install mssql-scripter wheel package locally.
+    """
+    root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
+    # Local install of mssql-scripter.
+    mssqlscripter_wheel_dir = os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
+    current_platform = utility.get_current_platform()
 
-    if len(options) >= 1:
-        if options[0] not in supported_actions:
-            print('Please provide a supported action {}.'.format(supported_actions))
-            return
-        action = options[0]
+    mssqlscripter_wheel_name = [pkge for pkge in mssqlscripter_wheel_dir if current_platform in pkge]
 
-    if action == 'nightly':
-        assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable'
-
-    print_heading('Cleanup')
-
-    # clean
-    utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
-    utility.clean_up(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
-    utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
-    utility.cleaun_up_egg_info_sub_directories(utility.MSSQLTOOLSSERVICE_DIRECTORY)
-    
-    print_heading('Running setup')
-    
-    # install general requirements.
-    utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
-    
-    print_heading('Running mssql-scripter tests')
-    utility.exec_command('tox', utility.ROOT_DIR, continue_on_error = False)
+    # To ensure we have a clean install, we disable the cache as to prevent cache overshadowing actual changes made.
+    utility.exec_command(
+       'pip install --no-cache-dir --no-index ./dist/{}'.format(mssqlscripter_wheel_name),
+       root_dir, continue_on_error=False)
 
-    print_heading('Building mssql-scripter pip package')
-    utility.exec_command('python setup.py check -r -s sdist', utility.ROOT_DIR, continue_on_error = False)
-    
-    print_heading('Building mssqltoolsservice pip package')
-    utility.exec_command('python buildwheels.py', utility.MSSQLTOOLSSERVICE_DIRECTORY, continue_on_error = False)
 
-    if action == 'nightly':
-        blob_service = BlockBlobService(connection_string=AZURE_STORAGE_CONNECTION_STRING)
-    
-        print_heading('Uploading packages to blob storage ')
-        for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY):
-            pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg)
-            print('Uploading package {}'.format(pkg_path))
-            upload_package(blob_service, pkg_path, 'mssql-scripter')
-    
-        for pkg in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY):
-            pkg_path = os.path.join(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY, pkg)
-            pkg_name = os.path.basename(pkg_path).split('-')[0].replace('_', '-').lower()
-            print('Uploading package {}'.format(pkg_name))
-            upload_package(blob_service, pkg_path, pkg_name)
-
-        # Upload the final index file
-        upload_index_file(blob_service, 'index.html', 'Simple Index', UPLOADED_PACKAGE_LINKS)
+def publish_daily(platforms_names):
+    """
+    Publish mssql-scripter wheel package to daily storage account.
+    """
+    print('Publishing to simple container within storage account.')
+    assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable'
+
+    blob_service = BlockBlobService(connection_string=AZURE_STORAGE_CONNECTION_STRING)
+
+    print_heading('Uploading packages to blob storage ')
+    for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY):
+        pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg)
+        print('Uploading package {}'.format(pkg_path))
+        _upload_package(blob_service, pkg_path, 'mssql-scripter')
+        
+    # Upload index files
+    _gen_pkg_index_html(blob_service, 'mssql-scripter')
+    _upload_index_file(blob_service, 'index.html', 'Simple Index', UPLOADED_PACKAGE_LINKS)
+
+
+def publish_official(platforms_names):
+    """
+    Publish mssql-scripter wheel package to PyPi.
+    """
+    mssqlscripter_wheel_dir = os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
+    # Run twine action for mssqlscripter.
+    # Only authorized users with credentials will be able to upload this package.
+    # Credentials will be stored in a .pypirc file.
+    for mssqlscripter_wheel_name in mssqlscripter_wheel_dir:
+        utility.exec_command(
+            'twine upload {} pypi'.format(mssqlscripter_wheel_name),
+            utility.MSSQLSCRIPTER_DIST_DIRECTORY)
 
 
 if __name__ == '__main__':
-    build(sys.argv[1:])
+    action = 'build'
+    supported_platforms = [
+        'win32',
+        'win_amd64',
+        'win64',
+        'macosx_10_11_intel',
+        'manylinux1_x86_64',
+        'manylinux1_i686']
+
+    targets = {
+        'build': build,
+        'validate_package': validate_package,
+        'publish_daily': publish_daily,
+        'publish_official': publish_official
+    }
+
+    if len(sys.argv) > 1:
+        action = sys.argv[1]
+    
+    if len(sys.argv) > 2:
+        supported_platforms = [sys.argv[2]]
+    
+    if action in targets:
+        targets[action](supported_platforms)
+    else:
+        print('{} is not a supported action'.format(action))
+        print('Supported actions are {}'.format(list(targets.keys())))

dev_setup.py

@@ -9,19 +9,21 @@
 from __future__ import print_function
 
 import os
-import setup
+import platform
 import utility
+import mssqlscripter.mssqltoolsservice.external as mssqltoolsservice
 
 print('Running dev setup...')
 print('Root directory \'{}\'\n'.format(utility.ROOT_DIR))
 
 # install general requirements.
 utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
+run_time_id = utility.get_current_platform()
+
+if run_time_id:
+    mssqltoolsservice.copy_sqltoolsservice(run_time_id)
+else:
+    print("This platform does not support mssqltoolsservice.")
 
-# install mssqltoolsservice if this platform supports it.
-mssqltoolsservice_package_name = os.environ['MSSQLTOOLSSERVICE_PACKAGE_NAME']
-print('Installing {}...'.format(mssqltoolsservice_package_name))
-# mssqltoolsservice package name is retrieved from environment variable set by setup.py.
-utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), utility.ROOT_DIR)
 
 print('Finished dev setup.')

doc/pypi_release_steps.md

@@ -40,37 +40,25 @@ bumpversion release_version    ->  1.0.0a<b>1</b>
     ##### Windows
       ```
       rmdir /s dist
-      rmdir /s mssqltoolsservice\dist
       ```
   
     ##### OSX/Ubuntu (bash)
       ```
       rm -rf dist
-      rm -rf mssqltoolsservice/dist
       ```
-2. Build mssql-scripter source distribution and verify readme.rst, From `<clone_root>` execute:
+2. Build mssql-scripter platform wheels and verify readme.rst, From `<clone_root>` execute:
     ```
-    python setup.py check -r -s sdist
+    python build.py
     ```
-
-3. Build mssqltoolsservice wheels for each supported platform, From `<clone_root>/mssqltoolsservice` execute:
-    ```
-	python buildwheels.py
-	```
-
-	Build a OS-Specific wheel:
+    
+    Build a OS-Specific wheel:
 	```
-    python buildwheels.py CentOS_7
-    python buildwheels.py Debian_8
-    python buildwheels.py Fedora_23
-    python buildwheels.py openSUSE_13_2
-    python buildwheels.py OSX_10_11_64
-    python buildwheels.py RHEL_7
-    python buildwheels.py Ubuntu_14
-    python buildwheels.py Ubuntu_16
-    python buildwheels.py Windows_7_64
-    python buildwheels.py Windows_7_86
+    python build.py build win32
+    python build.py build win_amd64
+    python build.py build macosx_10_11_intel
+    python build.py build manylinux1_x86_64
 	```
+
 4. Add a .pypirc configuration file:
 
     - Create a .pypirc file in your user directory:
@@ -97,9 +85,9 @@ bumpversion release_version    ->  1.0.0a<b>1</b>
 
 5. Test install locally
 
-	To install the local mssql-scripter pip package, from `<clone_root>` execute:
+	To install the local mssql-scripter wheel package, from `<clone_root>` execute:
     ```
-    sudo pip install --no-index -i ./mssqltoolsservice/dist/* ./dist/mssql-scripter-1.0.0a1.tar.gz
+    sudo pip install --no-index -i ./dist/mssql_scripter-1.0.0a1-py2.py3-none-win32.whl
     ```
 
 6. Test install via pypi server:

dos2unix.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+"""\
+convert dos linefeeds (crlf) to unix (lf)
+usage: dos2unix.py <input> <output>
+"""
+
+__version__ = "1"  # version is needed for packaging
+
+import sys
+
+if len(sys.argv[1:]) != 2:
+  sys.exit(__doc__)
+
+content = ''
+outsize = 0
+with open(sys.argv[1], 'rb') as infile:
+  content = infile.read()
+with open(sys.argv[2], 'wb') as output:
+  for line in content.splitlines():
+    outsize += len(line) + 1
+    output.write(line + b'\n')
+
+print("Done. Stripped %s bytes." % (len(content)-outsize))

mssqlscripter/__init__.py

@@ -4,4 +4,4 @@
 # --------------------------------------------------------------------------------------------
 
 
-__version__ = '1.0.0a21'
+__version__ = '1.0.0a22'

mssqlscripter/main.py

@@ -18,7 +18,7 @@
 import mssqlscripter.argparser as parser
 import mssqlscripter.scriptercallbacks as scriptercallbacks
 import mssqlscripter.sqltoolsclient as sqltoolsclient
-import mssqltoolsservice
+import mssqlscripter.mssqltoolsservice as mssqltoolsservice
 
 logger = logging.getLogger(u'mssqlscripter.main')