[AIRFLOW-5204] Shellcheck + common licences + executable shebangs in shell files (apache#5807)

* [AIRFLOW-5204] Shellcheck + common licence in shell files

Author: potiuk

.pre-commit-config.yaml

@@ -65,6 +65,19 @@ repos:
           - "/**| *| */"
           - --license-filepath
           - license-templates/LICENSE.txt
+      - id: insert-license
+        name: Add licence for shell files
+        exclude: ^\.github/.*$"|^airflow/_vendor/.*$
+        types: [shell]
+        args:
+          - --comment-style
+          - "|#|"
+          - --license-filepath
+          - license-templates/LICENSE.txt
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-executables-have-shebangs
   - repo: meta
     hooks:
       - id: check-hooks-apply
@@ -77,6 +90,11 @@ repos:
       - id: mixed-line-ending
   - repo: local
     hooks:
+      - id: shellcheck
+        name: Check Shell scripts syntax correctness
+        language: docker_image
+        entry: koalaman/shellcheck:stable -x -a
+        types: [shell]
       - id: lint-dockerfile
         name: Lint dockerfile
         language: system

CONTRIBUTING.md

@@ -698,15 +698,19 @@ you that you should rebuild the image with REBUILD="true" environment variable s
 In airflow we have the following checks:
 
 ```text
-lint-dockerfile                  Lint dockerfile
-mypy                             Run mypy
-pylint                           Run pylint
-flake8                           Run flake8
-forbid-tabs                      Fails if tabs are used in the project
 check-merge-conflict             Checks if merge conflict is being committed
+check-executables-have-shebangs  Check that executables have shebang
+check-hooks-apply                Check hooks apply to the repository
 detect-private-key               Detects if private key is added to the repository
 end-of-file-fixer                Make sure that there is an empty line at the end
+flake8                           Run flake8
+forbid-tabs                      Fails if tabs are used in the project
+insert-license                   Add licences for shell file types
+lint-dockerfile                  Lint dockerfile
 mixed-line-ending                Detects if mixed line ending is used (\r vs. \r\n)
+mypy                             Run mypy
+pylint                           Run pylint
+shellcheck                       Check shell files with shellcheck
 ```
 ## Using pre-commit hooks
 

airflow/bin/airflow_scheduler_autorestart.sh

@@ -1,14 +1,14 @@
-#
+#!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

airflow/contrib/hooks/qubole_hook.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -81,7 +80,13 @@
 #         network_mode='bridge',
 #         volumes=['/your/host/input_dir/path:/your/input_dir/path',
 #                  '/your/host/output_dir/path:/your/output_dir/path'],
-#         command='./entrypoint.sh',
+#         command=[
+#             "/bin/bash",
+#             "-c",
+#             "/bin/sleep 30; "
+#             "/bin/mv {{params.source_location}}/{{ ti.xcom_pull('view_file') }} {{params.target_location}};"
+#             "/bin/echo '{{params.target_location}}/{{ ti.xcom_pull('view_file') }}';"
+#         ],
 #         task_id='move_data',
 #         do_xcom_push=True,
 #         params={'source_location': '/your/input_dir/path',

airflow/contrib/hooks/ssh_hook.py

@@ -1,24 +1,25 @@
-#!/bin/sh
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 set -e
 
 # first bump up package.json manually, commit and tag
-if [ -d airflow/www/static/dist ]; then
+if [[ -d airflow/www/static/dist ]]; then
   rm -f airflow/www/static/dist/*
 fi
 

airflow/contrib/operators/bigquery_to_mysql_operator.py

@@ -1,28 +1,28 @@
 #!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # Use this to sign the tar balls generated from
 # python setup.py sdist --formats=gztar
 # ie. sign.sh <my_tar_ball>
 # you will still be required to type in your signing key password
 # or it needs to be available in your keychain
 
-NAME=${1}
+NAME="${1}"
 
-gpg --armor --output ${NAME}.asc --detach-sig ${NAME}
-gpg --print-md SHA512 ${NAME} > ${NAME}.sha512
+gpg --armor --output "${NAME}.asc" --detach-sig "${NAME}"
+gpg --print-md SHA512 "${NAME}" > "${NAME}.sha512"

airflow/contrib/operators/qubole_operator.py

@@ -1,16 +1,14 @@
 #!/usr/bin/env bash
-
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -40,7 +38,7 @@ if [[ -f /.dockerenv ]]; then
     sudo mkdir -pv _api
     echo "Created the _build and _api folders in case they do not exist"
     echo "Changing ownership of _build and _api folders to ${AIRFLOW_USER}:${AIRFLOW_USER}"
-    sudo chown -R "${AIRFLOW_USER}:${AIRFLOW_USER}" .
+    sudo chown -R "${AIRFLOW_USER}":"${AIRFLOW_USER}" .
     echo "Changed ownership of the whole doc folder to ${AIRFLOW_USER}:${AIRFLOW_USER}"
 else
     # We are outside the container so we simply make sure that the directories exist
@@ -86,23 +84,23 @@ SUCCEED_LINE=$(make html |\
     grep 'build succeeded' |\
     head -1)
 
-NUM_CURRENT_WARNINGS=$(echo ${SUCCEED_LINE} |\
+NUM_CURRENT_WARNINGS=$(echo "${SUCCEED_LINE}" |\
     sed -E 's/build succeeded, ([0-9]+) warnings?\./\1/g')
 
 if [[  -f /.dockerenv ]]; then
     # We are inside the container which means that we should fix back the permissions of the
     # _build and _api folder files, so that they can be accessed by the host user
     # The _api folder should be deleted by then but just in case we should change the ownership
     echo "Changing ownership of docs/_build folder back to ${HOST_USER_ID}:${HOST_GROUP_ID}"
-    sudo chown ${HOST_USER_ID}:${HOST_GROUP_ID} _build
+    sudo chown "${HOST_USER_ID}":"${HOST_GROUP_ID}" _build
     if [[ -d _api ]]; then
-        sudo chown ${HOST_USER_ID}:${HOST_GROUP_ID} _api
+        sudo chown "${HOST_USER_ID}":"${HOST_GROUP_ID}" _api
     fi
     echo "Changed ownership of docs/_build folder back to ${HOST_USER_ID}:${HOST_GROUP_ID}"
 fi
 
 
-if echo ${SUCCEED_LINE} | grep -q "warning"; then
+if echo "${SUCCEED_LINE}" | grep -q "warning"; then
     echo
     echo "Unexpected problems found in the documentation. "
     echo "Currently, ${NUM_CURRENT_WARNINGS} warnings found. "

airflow/example_dags/docker_copy_data.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env bash
-
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -18,8 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-(cd $THIS_DIR/_build/html;
+MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+(cd "${MY_DIR}"/_build/html || exit;
 # The below command works on both Python 2 and Python 3
 python -m http.server 8000 && python -m SimpleHTTPServer 8000
 )

airflow/example_dags/entrypoint.sh

@@ -1,2 +1,19 @@
 #!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 export DEFAULT_BRANCH="master"

airflow/models/__init__.py

@@ -1,22 +1,20 @@
 #!/usr/bin/env bash
-
-#
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#  Unless required by applicable law or agreed to in writing,
-#  software distributed under the License is distributed on an
-#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-#  KIND, either express or implied.  See the License for the
-#  specific language governing permissions and limitations
-#  under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # This is hook build used by DockerHub. We are also using it
 # on Travis CI to potentially rebuild (and refresh layers that
@@ -158,7 +156,7 @@ export PYTHON_VERSION=\
 ${PYTHON_VERSION:=$(${PYTHON_BINARY} -c \
  'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')}
 
-# shellcheck source=./_default_branch.sh
+# shellcheck source=hooks/_default_branch.sh
 . "${MY_DIR}/_default_branch.sh"
 
 # Source brnnch will be set in DockerHub

airflow/models/taskfail.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,6 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
 MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)