diff --git a/adhocracy4/follows/static/follows/FollowButton.jsx b/adhocracy4/follows/static/follows/FollowButton.jsx
index 11257e36f..f38389c89 100644
--- a/adhocracy4/follows/static/follows/FollowButton.jsx
+++ b/adhocracy4/follows/static/follows/FollowButton.jsx
@@ -12,7 +12,7 @@ const translated = {
   followingDescription: django.gettext(
     'Click to no longer be updated about this project via email.'
   ),
-  followAlert: django.gettext('From now on, we\'ll keep you updated on all changes. Make sure email notifications are enabled in your %(linkStart)s profile settings%(linkEnd)s'),
+  followAlert: django.gettext('From now on, we\'ll keep you updated on all changes.<br/>Make sure email notifications are enabled in your %(linkStart)s profile settings%(linkEnd)s'),
   followingAlert: django.gettext('You will no longer be updated via email.'),
   follow: django.gettext('Follow'),
   following: django.gettext('Following')
diff --git a/adhocracy4/follows/static/follows/__tests__/FollowButton.jest.jsx b/adhocracy4/follows/static/follows/__tests__/FollowButton.jest.jsx
index 1cd2ce80c..cd5ba92e5 100644
--- a/adhocracy4/follows/static/follows/__tests__/FollowButton.jest.jsx
+++ b/adhocracy4/follows/static/follows/__tests__/FollowButton.jest.jsx
@@ -88,7 +88,11 @@ test('Test AlertPortal renders correctly with existing target', async () => {
   render(<FollowButton authenticatedAs project="test" alertTarget="alert-container" />)
   const followButton = await screen.findByText('Follow')
   fireEvent.click(followButton)
-  const alertElement = screen.getByText('From now on, we\'ll keep you updated on all changes. Make sure email notifications are enabled in your %(linkStart)s profile settings%(linkEnd)s')
+  const alertElement = screen.getByText((content, element) => {
+    return content.includes('From now on, we\'ll keep you updated on all changes') &&
+           content.includes('Make sure email notifications are enabled in your') &&
+           content.includes('profile settings')
+  })
   expect(alertElement).toBeInTheDocument()
   document.body.removeChild(alertContainer)
 })
diff --git a/adhocracy4/static/Alert.jsx b/adhocracy4/static/Alert.jsx
index 12ee6a7ac..add2dca2a 100644
--- a/adhocracy4/static/Alert.jsx
+++ b/adhocracy4/static/Alert.jsx
@@ -34,17 +34,17 @@ const Alert = ({ type = 'info', title, message, htmlMessage, onClick, timeInMs }
         {htmlMessage
           ? (<div dangerouslySetInnerHTML={{ __html: htmlMessage }} />)
           : (message)}
-        {onClick && (
-          <button
-            type="button"
-            className="alert__close"
-            aria-label={closeTag}
-            onClick={onClick}
-          >
-            <span className="fa fa-times" aria-hidden="true" />
-          </button>
-        )}
       </div>
+      {onClick && (
+        <button
+          type="button"
+          className="alert__close"
+          aria-label={closeTag}
+          onClick={onClick}
+        >
+          <span className="fa fa-times" aria-hidden="true" />
+        </button>
+      )}
     </div>
   )
 }