try to fix github oom error on test

graphsense · Nov 19, 2024 · df210aa · df210aa
1 parent ae6ea0f
commit df210aa
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 48 deletions.
diff --git a/build.sbt b/build.sbt
@@ -87,5 +87,5 @@ lazy val root = (project in file(".")).
       "org.apache.spark" %% "spark-sql" % "3.5.3" % Provided,
       "org.apache.spark" %% "spark-graphx" % "3.5.3" % Provided,
       "graphframes" % "graphframes" % "0.8.3-spark3.5-s_2.12" % Provided),
-    javaOptions ++= Seq("-Xms10g", "-Xmx10g", "-XX:+CMSClassUnloadingEnabled")
+    javaOptions ++= Seq("-Xms8g", "-Xmx8g", "-XX:+CMSClassUnloadingEnabled")
   )
diff --git a/src/test/scala/org/graphsense/utxo/TransformationTest.scala b/src/test/scala/org/graphsense/utxo/TransformationTest.scala
@@ -49,6 +49,16 @@ class TransformationTest extends TestBase {
   val addressIds = t.computeAddressIds(regOutputs)
   val addressByAddressPrefix = t.computeAddressByAddressPrefix(addressIds)
 
+  test("addressByAddressPrefix") {
+    val addressByAddressPrefixRef =
+      readTestData[AddressByAddressPrefix](
+        refDir + "address_by_address_prefix.json"
+      )
+    assertDataFrameEquality(addressByAddressPrefix, addressByAddressPrefixRef)
+  }
+
+  addressByAddressPrefix.unpersist()
+
   val addressTransactions =
     t.computeAddressTransactions(
       regInputs,
@@ -57,6 +67,15 @@ class TransformationTest extends TestBase {
     ).sort(F.addressId, F.blockId, F.value)
       .persist()
 
+  test("regularOutputs") {
+    val regOutputsRef =
+      readTestData[RegularOutput](refDir + "regular_outputs.json")
+        .sort(F.txId, F.address)
+    val sortedOutput = regOutputs.sort(F.txId, F.address)
+    assertDataFrameEquality(sortedOutput, regOutputsRef)
+  }
+  regOutputs.unpersist()
+
   val (inputs, outputs) = t.splitTransactions(addressTransactions)
   inputs.persist()
   outputs.persist()
@@ -94,10 +113,34 @@ class TransformationTest extends TestBase {
       .persist()
   val noAddresses = addresses.count()
 
+  test("basicAddresses") {
+    val basicAddressesRef =
+      readTestData[BasicAddress](refDir + "basic_addresses.json")
+    assertDataFrameEquality(basicAddresses, basicAddressesRef)
+  }
+  basicAddresses.unpersist()
+
   val addressClusterCoinjoin =
     t.computeAddressCluster(regInputs, addressIds, false)
       .sort(F.addressId)
       .persist()
+
+  test("regularInputs") {
+    val regInputsRef =
+      readTestData[RegularInput](refDir + "regular_inputs.json")
+        .sort(F.txId, F.address)
+    val sortedInputs = regInputs.sort(F.txId, F.address)
+    assertDataFrameEquality(sortedInputs, regInputsRef)
+  }
+  regInputs.unpersist()
+
+  test("addressIds") {
+    val addressIdsRef =
+      readTestData[AddressId](refDir + "address_ids.json")
+    assertDataFrameEquality(addressIds, addressIdsRef)
+  }
+  addressIds.unpersist()
+
   val clusterAddresses =
     t.computeClusterAddresses(addressClusterCoinjoin)
       .sort(F.clusterId, F.addressId)
@@ -135,9 +178,27 @@ class TransformationTest extends TestBase {
   val noClusterRelations = clusterRelations.count()
 
   plainClusterRelations.unpersist()
+
+  test("clusterInputs") {
+    val clusterInputsRef =
+      readTestData[ClusterTransaction](refDir + "cluster_inputs.json")
+    assertDataFrameEquality(clusterInputs, clusterInputsRef)
+  }
   clusterInputs.unpersist()
+
+  test("clusterOutputs") {
+    val clusterOutputsRef =
+      readTestData[ClusterTransaction](refDir + "cluster_outputs.json")
+    assertDataFrameEquality(clusterOutputs, clusterOutputsRef)
+  }
   clusterOutputs.unpersist()
+
+  test("addresses") {
+    val addressesRef = readTestData[Address](refDir + "addresses.json")
+    assertDataFrameEquality(addresses, addressesRef)
+  }
   addresses.unpersist()
+
   val cluster =
     t.computeCluster(basicCluster, clusterRelations)
       .sort(F.clusterId)
@@ -155,35 +216,8 @@ class TransformationTest extends TestBase {
       noClusterRelations
     )
 
-  note("test address graph")
-
-  test("addressIds") {
-    val addressIdsRef =
-      readTestData[AddressId](refDir + "address_ids.json")
-    assertDataFrameEquality(addressIds, addressIdsRef)
-  }
-  test("addressByAddressPrefix") {
-    val addressByAddressPrefixRef =
-      readTestData[AddressByAddressPrefix](
-        refDir + "address_by_address_prefix.json"
-      )
-    assertDataFrameEquality(addressByAddressPrefix, addressByAddressPrefixRef)
-  }
+  // note("test address graph")
 
-  test("regularInputs") {
-    val regInputsRef =
-      readTestData[RegularInput](refDir + "regular_inputs.json")
-        .sort(F.txId, F.address)
-    val sortedInputs = regInputs.sort(F.txId, F.address)
-    assertDataFrameEquality(sortedInputs, regInputsRef)
-  }
-  test("regularOutputs") {
-    val regOutputsRef =
-      readTestData[RegularOutput](refDir + "regular_outputs.json")
-        .sort(F.txId, F.address)
-    val sortedOutput = regOutputs.sort(F.txId, F.address)
-    assertDataFrameEquality(sortedOutput, regOutputsRef)
-  }
   test("addressTransactions") {
     val addressTransactionsRef =
       readTestData[AddressTransaction](refDir + "address_txs.json")
@@ -199,20 +233,12 @@ class TransformationTest extends TestBase {
       readTestData[AddressTransaction](refDir + "outputs.json")
     assertDataFrameEquality(outputs, outputsRef)
   }
-  test("basicAddresses") {
-    val basicAddressesRef =
-      readTestData[BasicAddress](refDir + "basic_addresses.json")
-    assertDataFrameEquality(basicAddresses, basicAddressesRef)
-  }
+
   test("addressRelations") {
     val addressRelationsRef =
       readTestData[AddressRelation](refDir + "address_relations.json")
     assertDataFrameEquality(addressRelations, addressRelationsRef)
   }
-  test("addresses") {
-    val addressesRef = readTestData[Address](refDir + "addresses.json")
-    assertDataFrameEquality(addresses, addressesRef)
-  }
 
   note("test cluster graph")
 
@@ -233,16 +259,7 @@ class TransformationTest extends TestBase {
       readTestData[ClusterTransaction](refDir + "cluster_txs.json")
     assertDataFrameEquality(clusterTransactions, clusterTransactionsRef)
   }
-  test("clusterInputs") {
-    val clusterInputsRef =
-      readTestData[ClusterTransaction](refDir + "cluster_inputs.json")
-    assertDataFrameEquality(clusterInputs, clusterInputsRef)
-  }
-  test("clusterOutputs") {
-    val clusterOutputsRef =
-      readTestData[ClusterTransaction](refDir + "cluster_outputs.json")
-    assertDataFrameEquality(clusterOutputs, clusterOutputsRef)
-  }
+
   test("basicCluster") {
     val basicClusterRef =
       readTestData[BasicCluster](refDir + "basic_cluster.json")