Fix recursion detection

Author: nikolay-egorov

jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/VariableState.kt

@@ -39,6 +39,7 @@ data class VariableStateImpl(
         }
     }
     override var isRecursive: Boolean = false
+    private var isLargeForString: Boolean = false
 
     private val valCache = VariableStateCache<Result<Any?>> (
         {
@@ -73,17 +74,35 @@ data class VariableStateImpl(
             val res = action(this)
             isAccessible = wasAccessible
             return res
-    private val customDelegate = DependentLazyDelegate {
-        fun getRecursiveObjectName(): String {
-            val kClassName = cachedValue.getOrNull()!!::class.simpleName
-            return "$kClassName: recursive structure"
         }
-        if (cachedValue.getOrNull() == null) {
-            return@DependentLazyDelegate null
-        }
-        handleIfRecursiveStructure()
     }
-}
+            private val customDelegate = DependentLazyDelegate {
+                fun getRecursiveObjectName(): String {
+                    val kClassName = cachedValue.getOrNull()!!::class.simpleName
+                    return "$kClassName: recursive structure"
+                }
+                if (cachedValue.getOrNull() == null) {
+                    return@DependentLazyDelegate null
+                }
+                handleIfRecursiveStructure()
+                try {
+                    cachedValue.getOrNull().toString()
+                    isRecursive = false
+                    isLargeForString = false
+                } catch (e: VirtualMachineError) {
+                    when (e) {
+                        is StackOverflowError -> {
+                            isRecursive = true
+                        }
+                        is OutOfMemoryError -> {
+                            isLargeForString = true
+                        }
+                        else -> {
+                            return@DependentLazyDelegate null
+                        }
+                    }
+                }
+            }
 
 private class VariableStateCache<T>(
     val equalityChecker: (T, T) -> Boolean = { x, y -> x == y },

src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

@@ -774,6 +774,33 @@ class ReplVarsTest : AbstractSingleReplTest() {
         assertEquals(2, newData.fieldDescriptor.size)
     }
 
+    @Test
+    fun testProperBiRecursionHandling() {
+        eval(
+            """
+            val l = mutableListOf<Any>()
+            l.add(listOf(l))
+            
+            val m = mutableMapOf<Int, Any>(1 to l)
+            
+            val z = setOf(1, 2, 4)
+            """.trimIndent(),
+            jupyterId = 1
+        )
+        var state = repl.notebook.variablesState
+        assertEquals("ArrayList: recursive structure", state["l"]!!.stringValue)
+        assertEquals("LinkedHashMap: recursive structure", state["m"]!!.stringValue)
+        eval(
+            """
+            val m = mutableMapOf<Int, Any>(1 to "abc")
+            """.trimIndent(),
+            jupyterId = 2
+        )
+        state = repl.notebook.variablesState
+        assertEquals("ArrayList: recursive structure", state["l"]!!.stringValue)
+        assertNotEquals("LinkedHashMap: recursive structure", state["m"]!!.stringValue)
+    }
+
     @Test
     fun testUnchangedVars() {
         eval(