Merge pull request #777 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

.github/workflows/canary-integration-test.yml

@@ -70,6 +70,21 @@ jobs:
           mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
           timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
 
+      - name: test osd.0 auth recovery from keyring file
+        run: |
+          toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
+          osd_id=0
+          osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
+          if [ $osd_pod ]; then
+            timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth del osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be deleted'; done";
+            kubectl -n rook-ceph delete pod $osd_pod;
+            timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph auth get osd.$osd_id ; do sleep 1 && echo 'waiting for osd auth to be recovered'; done";
+            osd_pod=$(kubectl get pod -l app=rook-ceph-osd,osd=$osd_id -n rook-ceph -o jsonpath='{.items[*].metadata.name}');
+            kubectl -n rook-ceph wait --for=condition=Ready pod/$osd_pod --timeout=120s;
+          else
+            echo "osd $osd_id not found, skipping test";
+          fi
+
       - name: test external script create-external-cluster-resources.py
         run: |
           toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')

Documentation/CRDs/Cluster/external-cluster/provider-export.md

@@ -4,7 +4,7 @@ In order to configure an external Ceph cluster with Rook, we need to extract som
 
 ## 1. Create all users and keys
 
-Run the python script [create-external-cluster-resources.py](https://github.com/rook/rook/blob/master/deploy/examples/external/create-external-cluster-resources.py) for creating all users and keys.
+Run the python script [create-external-cluster-resources.py](https://github.com/rook/rook/blob/master/deploy/examples/external/create-external-cluster-resources.py) in the provider Ceph cluster cephadm shell, to have access to create the necessary users and keys.
 
 ```console
 python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --cephfs-filesystem-name <filesystem-name> --rgw-endpoint  <rgw-endpoint> --namespace <namespace> --format bash

pkg/operator/ceph/cluster/osd/spec.go

@@ -107,11 +107,42 @@ OSD_ID="$ROOK_OSD_ID"
 OSD_UUID=%s
 OSD_STORE_FLAG="%s"
 OSD_DATA_DIR=/var/lib/ceph/osd/ceph-"$OSD_ID"
+KEYRING_FILE="$OSD_DATA_DIR"/keyring
 CV_MODE=%s
 DEVICE="$%s"
 
-# create new keyring
-ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
+# In rare cases keyring file created with prepare-osd but did not
+# being stored in ceph auth system therefore we need to import it
+# from keyring file instead of creating new one
+if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then
+    if [ -f "$KEYRING_FILE" ]; then
+        # import keyring from existing file
+        TMP_DIR=$(mktemp -d)
+
+        python3 -c "
+import configparser
+
+config = configparser.ConfigParser()
+config.read('$KEYRING_FILE')
+
+if not config.has_section('osd.$OSD_ID'):
+    exit()
+
+config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'}
+
+with open('$TMP_DIR/keyring', 'w') as configfile:
+    config.write(configfile)
+"
+
+        cat "$TMP_DIR"/keyring
+        ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring
+
+        rm --recursive --force "$TMP_DIR"
+    else
+        # create new keyring if no keyring file found
+        ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
+    fi
+fi
 
 # active the osd with ceph-volume
 if [[ "$CV_MODE" == "lvm" ]]; then