Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
minmingzhu committed Jul 29, 2024
1 parent cba24c9 commit a9fb3b1
Showing 1 changed file with 13 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,19 @@ class CorrelationDALImpl(
val kvsIPPort = getOneCCLIPPort(coalescedTables)
val training_breakdown_name = "Correlation_training_breakdown_" + executorNum;

coalescedTables.mapPartitionsWithIndex { (rank, iter) =>
logInfo(s"set ZE_AFFINITY_MASK")
val gpuIndices = if (useDevice == "GPU") {
val resources = TaskContext.get().resources()
resources("gpu").addresses.map(_.toInt)
} else {
null
}
logInfo(s"set ZE_AFFINITY_MASK rank is $rank.")
logInfo(s"gpuIndices is ${gpuIndices.mkString(", ")}.")
OneCCL.setExecutorEnv("ZE_AFFINITY_MASK", gpuIndices(0).toString())
Iterator.empty
}.count()
// coalescedTables.mapPartitionsWithIndex { (rank, iter) =>
// logInfo(s"set ZE_AFFINITY_MASK")
// val gpuIndices = if (useDevice == "GPU") {
// val resources = TaskContext.get().resources()
// resources("gpu").addresses.map(_.toInt)
// } else {
// null
// }
// logInfo(s"set ZE_AFFINITY_MASK rank is $rank.")
// logInfo(s"gpuIndices is ${gpuIndices.mkString(", ")}.")
// OneCCL.setExecutorEnv("ZE_AFFINITY_MASK", gpuIndices(0).toString())
// Iterator.empty
// }.count()

if (useDevice == "CPU") {
coalescedTables.mapPartitionsWithIndex { (rank, table) =>
Expand Down

0 comments on commit a9fb3b1

Please sign in to comment.