diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 3a91522d36229..28a7598a7c195 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -4,7 +4,6 @@
 use crate::ty::{DefIdTree, TyCtxt, Visibility};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_hir::def::DefKind;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_span::def_id::LocalDefId;
@@ -167,7 +166,7 @@ impl EffectiveVisibilities {
             // and all effective visibilities are larger or equal than private visibility.
             let private_vis = Visibility::Restricted(tcx.parent_module_from_def_id(def_id));
             let span = tcx.def_span(def_id.to_def_id());
-            if !ev.direct.is_at_least(private_vis, tcx) {
+            if !ev.direct.is_at_least(private_vis, tcx) && tcx.sess.has_errors().is_none() {
                 span_bug!(span, "private {:?} > direct {:?}", private_vis, ev.direct);
             }
             if !ev.reexported.is_at_least(ev.direct, tcx) {
@@ -185,7 +184,6 @@ impl EffectiveVisibilities {
                 );
             }
             let nominal_vis = tcx.visibility(def_id);
-            let def_kind = tcx.opt_def_kind(def_id);
             // FIXME: `rustc_privacy` is not yet updated for the new logic and can set
             // effective visibilities that are larger than the nominal one.
             if !nominal_vis.is_at_least(ev.reachable_through_impl_trait, tcx) && early {
@@ -197,11 +195,6 @@ impl EffectiveVisibilities {
                     nominal_vis
                 );
             }
-            // Fully private items are never put into the table, this is important for performance.
-            // FIXME: Fully private `mod` items are currently put into the table.
-            if ev.reachable_through_impl_trait == private_vis && def_kind != Some(DefKind::Mod) {
-                span_bug!(span, "fully private item in the table {:?}: {:?}", def_id, ev.direct);
-            }
         }
     }
 }
diff --git a/src/test/ui/privacy/effective_visibilities_invariants.rs b/src/test/ui/privacy/effective_visibilities_invariants.rs
new file mode 100644
index 0000000000000..af5a2bed6ab24
--- /dev/null
+++ b/src/test/ui/privacy/effective_visibilities_invariants.rs
@@ -0,0 +1,12 @@
+// Invariant checking doesn't ICE in some cases with errors (issue #104249).
+
+#![feature(staged_api)] //~ ERROR module has missing stability attribute
+
+pub mod m {} //~ ERROR module has missing stability attribute
+
+pub mod m { //~ ERROR the name `m` is defined multiple times
+    mod inner {}
+    type Inner = u8;
+}
+
+fn main() {}
diff --git a/src/test/ui/privacy/effective_visibilities_invariants.stderr b/src/test/ui/privacy/effective_visibilities_invariants.stderr
new file mode 100644
index 0000000000000..fd205f4058ae0
--- /dev/null
+++ b/src/test/ui/privacy/effective_visibilities_invariants.stderr
@@ -0,0 +1,32 @@
+error[E0428]: the name `m` is defined multiple times
+  --> $DIR/effective_visibilities_invariants.rs:7:1
+   |
+LL | pub mod m {}
+   | --------- previous definition of the module `m` here
+LL |
+LL | pub mod m {
+   | ^^^^^^^^^ `m` redefined here
+   |
+   = note: `m` must be defined only once in the type namespace of this module
+
+error: module has missing stability attribute
+  --> $DIR/effective_visibilities_invariants.rs:3:1
+   |
+LL | / #![feature(staged_api)]
+LL | |
+LL | | pub mod m {}
+LL | |
+...  |
+LL | |
+LL | | fn main() {}
+   | |____________^
+
+error: module has missing stability attribute
+  --> $DIR/effective_visibilities_invariants.rs:5:1
+   |
+LL | pub mod m {}
+   | ^^^^^^^^^^^^
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0428`.