New CSV implementation

.github/workflows/generated-sources-master.yml

@@ -26,7 +26,7 @@ jobs:
         run: |
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-          git add './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
+          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
           git diff --staged --quiet || git commit -m "Automated commit of generated code"
           git push
         env:

.github/workflows/generated-sources.yml

@@ -38,18 +38,18 @@ jobs:
           git config --global user.name "GitHub Actions"
 
       - name: Run Gradle task
-        run: ./gradlew :core:processKDocsMain korro
+        run: ./gradlew processKDocsMain korro
 
       - name: Check for changes in generated sources
         id: git-diff
-        run: echo "changed=$(if git diff --quiet './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
+        run: echo "changed=$(if git diff --quiet './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
 
       - name: Commit and push if changes
         id: git-commit
         if: steps.git-diff.outputs.changed == 'true'
         run: |
           git checkout -b generated-sources/docs-update-${{ github.run_number }}
-          git add './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
+          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
           git commit -m "Update generated sources with recent changes"
           git push origin generated-sources/docs-update-${{ github.run_number }}
           echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

build.gradle.kts

@@ -55,12 +55,15 @@ dependencies {
     api(project(":dataframe-excel"))
     api(project(":dataframe-openapi"))
     api(project(":dataframe-jdbc"))
+    // TODO enable when it leaves the experimental phase
+    //  api(project(":dataframe-csv"))
 
     kover(project(":core"))
     kover(project(":dataframe-arrow"))
     kover(project(":dataframe-excel"))
     kover(project(":dataframe-openapi"))
     kover(project(":dataframe-jdbc"))
+    kover(project(":dataframe-csv"))
     kover(project(":plugins:kotlin-dataframe"))
 }
 

core/api/core.api

@@ -6485,30 +6485,37 @@ public final class org/jetbrains/kotlinx/dataframe/api/ParseKt {
 }
 
 public final class org/jetbrains/kotlinx/dataframe/api/ParserOptions {
+	public static final field Companion Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions$Companion;
 	public fun <init> ()V
 	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;)V
 	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Z)V
-	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Z)V
+	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public final fun component1 ()Ljava/util/Locale;
 	public final fun component2 ()Ljava/time/format/DateTimeFormatter;
 	public final fun component3 ()Ljava/lang/String;
 	public final fun component4 ()Ljava/util/Set;
-	public final fun component5 ()Z
+	public final fun component5 ()Ljava/util/Set;
+	public final fun component6 ()Z
 	public final synthetic fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
-	public final fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Z)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
+	public final fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Z)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
 	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
-	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
+	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
 	public fun equals (Ljava/lang/Object;)Z
 	public final fun getDateTimeFormatter ()Ljava/time/format/DateTimeFormatter;
 	public final fun getDateTimePattern ()Ljava/lang/String;
 	public final fun getLocale ()Ljava/util/Locale;
 	public final fun getNullStrings ()Ljava/util/Set;
+	public final fun getSkipTypes ()Ljava/util/Set;
 	public final fun getUseFastDoubleParser ()Z
 	public fun hashCode ()I
 	public fun toString ()Ljava/lang/String;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/api/ParserOptions$Companion {
+	public final fun allTypesExcept ([Lkotlin/reflect/KType;)Ljava/util/Set;
+}
+
 public abstract interface class org/jetbrains/kotlinx/dataframe/api/Pivot : org/jetbrains/kotlinx/dataframe/aggregation/Aggregatable {
 }
 
@@ -9946,6 +9953,16 @@ public final class org/jetbrains/kotlinx/dataframe/impl/ColumnAccessTrackerKt {
 	public static final fun trackColumnAccess (Lkotlin/jvm/functions/Function0;)Ljava/util/List;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/impl/ColumnNameGenerator {
+	public fun <init> ()V
+	public fun <init> (Ljava/util/List;)V
+	public synthetic fun <init> (Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun addIfAbsent (Ljava/lang/String;)V
+	public final fun addUnique (Ljava/lang/String;)Ljava/lang/String;
+	public final fun contains (Ljava/lang/String;)Z
+	public final fun getNames ()Ljava/util/List;
+}
+
 public final class org/jetbrains/kotlinx/dataframe/impl/DataFrameSize {
 	public fun <init> (II)V
 	public final fun component1 ()I
@@ -10271,23 +10288,38 @@ public final class org/jetbrains/kotlinx/dataframe/io/CSVType : java/lang/Enum {
 public final class org/jetbrains/kotlinx/dataframe/io/ColType : java/lang/Enum {
 	public static final field BigDecimal Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field Boolean Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field Char Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field Companion Lorg/jetbrains/kotlinx/dataframe/io/ColType$Companion;
+	public static final field DEFAULT Ljava/lang/String;
 	public static final field Double Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field Duration Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field Instant Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field Int Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field JsonArray Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field JsonObject Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field LocalDate Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field LocalDateTime Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field LocalTime Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field Long Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static final field String Lorg/jetbrains/kotlinx/dataframe/io/ColType;
+	public static final field Url Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static fun getEntries ()Lkotlin/enums/EnumEntries;
 	public static fun valueOf (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 	public static fun values ()[Lorg/jetbrains/kotlinx/dataframe/io/ColType;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/io/ColType$Companion {
+}
+
 public final class org/jetbrains/kotlinx/dataframe/io/CommonKt {
 	public static final fun asFileOrNull (Ljava/net/URL;)Ljava/io/File;
+	public static final fun asUrl (Ljava/lang/String;)Ljava/net/URL;
+	public static final fun catchHttpResponse (Ljava/net/URL;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 	public static final fun isFile (Ljava/net/URL;)Z
 	public static final fun isProtocolSupported (Ljava/net/URL;)Z
 	public static final fun isURL (Ljava/lang/String;)Z
+	public static final fun isUrl (Ljava/lang/String;)Z
+	public static final fun skippingBomCharacters (Ljava/io/InputStream;)Ljava/io/InputStream;
 	public static final fun toDataFrame (Ljava/util/List;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 	public static synthetic fun toDataFrame$default (Ljava/util/List;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 	public static final fun urlAsFile (Ljava/net/URL;)Ljava/io/File;
@@ -10313,6 +10345,7 @@ public final class org/jetbrains/kotlinx/dataframe/io/CsvKt {
 	public static synthetic fun readDelimStr$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/String;CLjava/util/Map;ILjava/lang/Integer;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 	public static final fun toCsv (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/apache/commons/csv/CSVFormat;)Ljava/lang/String;
 	public static synthetic fun toCsv$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/apache/commons/csv/CSVFormat;ILjava/lang/Object;)Ljava/lang/String;
+	public static final fun toKType (Lorg/jetbrains/kotlinx/dataframe/io/ColType;)Lkotlin/reflect/KType;
 	public static final fun toType (Lorg/jetbrains/kotlinx/dataframe/io/ColType;)Lkotlin/reflect/KClass;
 	public static final fun writeCSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Ljava/io/File;Lorg/apache/commons/csv/CSVFormat;)V
 	public static final fun writeCSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Ljava/lang/Appendable;Lorg/apache/commons/csv/CSVFormat;)V

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -47,6 +47,7 @@ public enum class DataSchemaVisibility {
     EXPLICIT_PUBLIC,
 }
 
+// TODO add more options
 public annotation class CsvOptions(public val delimiter: Char)
 
 /**

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/parse.kt

@@ -16,6 +16,7 @@ import org.jetbrains.kotlinx.dataframe.util.PARSER_OPTIONS_COPY
 import java.time.format.DateTimeFormatter
 import java.util.Locale
 import kotlin.reflect.KProperty
+import kotlin.reflect.KType
 
 public val DataFrame.Companion.parser: GlobalParserOptions get() = Parsers
 
@@ -56,6 +57,8 @@ public interface GlobalParserOptions {
  *   it will be used to create a [DateTimeFormatter].
  * @param nullStrings a set of strings that should be treated as `null` values. By default, it's
  *   ["null", "NULL", "NA", "N/A"].
+ * @param skipTypes a set of types that should be skipped during parsing. Parsing will be attempted for all other types.
+ *   By default, it's an empty set. To skip all types except some specified ones, use [allTypesExcept].
  * @param useFastDoubleParser whether to use the new _experimental_ FastDoubleParser, defaults to `false` for now.
  */
 public data class ParserOptions(
@@ -64,8 +67,17 @@ public data class ParserOptions(
     val dateTimeFormatter: DateTimeFormatter? = null,
     val dateTimePattern: String? = null,
     val nullStrings: Set<String>? = null,
+    val skipTypes: Set<KType> = emptySet(),
     val useFastDoubleParser: Boolean = false,
 ) {
+    public companion object {
+        /**
+         * Small helper function to get all types except the ones specified.
+         * Useful in combination with the [skipTypes] parameter.
+         */
+        public fun allTypesExcept(vararg types: KType): Set<KType> =
+            Parsers.parsersOrder.map { it.type }.toSet() - types.toSet()
+    }
 
     /** For binary compatibility. */
     @Deprecated(
@@ -82,6 +94,7 @@ public data class ParserOptions(
         dateTimeFormatter = dateTimeFormatter,
         dateTimePattern = dateTimePattern,
         nullStrings = nullStrings,
+        skipTypes = emptySet(),
         useFastDoubleParser = false,
     )
 
@@ -101,6 +114,7 @@ public data class ParserOptions(
             dateTimeFormatter = dateTimeFormatter,
             dateTimePattern = dateTimePattern,
             nullStrings = nullStrings,
+            skipTypes = skipTypes,
             useFastDoubleParser = useFastDoubleParser,
         )
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnNameGenerator.kt

@@ -2,13 +2,13 @@ package org.jetbrains.kotlinx.dataframe.impl
 
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 
-internal class ColumnNameGenerator(columnNames: List<String> = emptyList()) {
+public class ColumnNameGenerator(columnNames: List<String> = emptyList()) {
 
     private val usedNames = columnNames.toMutableSet()
 
     private val colNames = columnNames.toMutableList()
 
-    fun addUnique(preferredName: String): String {
+    public fun addUnique(preferredName: String): String {
         var name = preferredName
         var k = 1
         while (usedNames.contains(name)) {
@@ -19,17 +19,17 @@ internal class ColumnNameGenerator(columnNames: List<String> = emptyList()) {
         return name
     }
 
-    fun addIfAbsent(name: String) {
+    public fun addIfAbsent(name: String) {
         if (!usedNames.contains(name)) {
             usedNames.add(name)
             colNames.add(name)
         }
     }
 
-    val names: List<String>
+    public val names: List<String>
         get() = colNames
 
-    fun contains(name: String) = usedNames.contains(name)
+    public operator fun contains(name: String): Boolean = usedNames.contains(name)
 }
 
 internal fun AnyFrame.nameGenerator() = ColumnNameGenerator(columnNames())

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

@@ -35,7 +35,7 @@ import org.jetbrains.kotlinx.dataframe.impl.catchSilent
 import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
 import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
 import org.jetbrains.kotlinx.dataframe.impl.javaDurationCanParse
-import org.jetbrains.kotlinx.dataframe.io.isURL
+import org.jetbrains.kotlinx.dataframe.io.isUrl
 import org.jetbrains.kotlinx.dataframe.io.readJsonStr
 import org.jetbrains.kotlinx.dataframe.values
 import java.math.BigDecimal
@@ -209,7 +209,7 @@ internal object Parsers : GlobalParserOptions {
         toJavaLocalDateTimeOrNull(formatter) // since we accept a Java DateTimeFormatter
             ?.toKotlinLocalDateTime()
 
-    private fun String.toUrlOrNull(): URL? = if (isURL(this)) catchSilent { URL(this) } else null
+    private fun String.toUrlOrNull(): URL? = if (isUrl(this)) catchSilent { URL(this) } else null
 
     private fun String.toBooleanOrNull() =
         when (uppercase(Locale.getDefault())) {
@@ -387,6 +387,8 @@ internal object Parsers : GlobalParserOptions {
                 null
             }
         },
+        // Char
+        stringParser<Char> { it.singleOrNull() },
         // No parser found, return as String
         // must be last in the list of parsers to return original unparsed string
         stringParser<String> { it },
@@ -460,7 +462,9 @@ internal fun DataColumn<String?>.tryParseImpl(options: ParserOptions?): DataColu
     var nullStringParsed = false
     val nulls = options?.nullStrings ?: Parsers.nulls
 
+    val parserTypesToSkip = options?.skipTypes ?: emptySet()
     val parsersToCheck = Parsers.parsersOrder
+        .filterNot { it.type in parserTypesToSkip }
     val parserTypesToCheck = parsersToCheck.map { it.type }.toSet()
 
     var correctParser: StringParser<*>? = null

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/FastDoubleParser.kt

@@ -43,10 +43,6 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
 
     private val parser = ConfigurableDoubleParser(/* symbols = */ setupNumberFormatSymbols(), /* ignoreCase = */ true)
 
-    // Fix for Java 8 RTL languages minus sign not being recognized
-    private val minusSignIsFormatSymbol =
-        Character.getType(localDecimalFormatSymbols.minusSign) == Character.FORMAT.toInt()
-
     /**
      * Sets up the [NumberFormatSymbols] for the [ConfigurableDoubleParser] based on
      * [localDecimalFormatSymbols] with fallbacks from [fallbackDecimalFormatSymbols].
@@ -181,17 +177,6 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     ): Double? {
         if (parserOptions.useFastDoubleParser && charset in supportedFastCharsets) {
             try {
-                // Fixes RTL minus sign not being recognized
-                if (minusSignIsFormatSymbol && ba.toString(charset).startsWith(localDecimalFormatSymbols.minusSign)) {
-                    val localMinusSize = localDecimalFormatSymbols.minusSign.toString().toByteArray(charset).size
-                    val fallbackMinusSize = fallbackDecimalFormatSymbols.minusSign.toString().toByteArray(charset).size
-                    val newOffset = (localMinusSize - fallbackMinusSize).coerceAtLeast(0)
-                    val newBa = ba.copyOf()
-                    fallbackDecimalFormatSymbols.minusSign.toString().toByteArray(charset)
-                        .copyInto(destination = newBa, destinationOffset = newOffset)
-
-                    return parser.parseDouble(newBa, newOffset, length - newOffset)
-                }
                 return parser.parseDouble(ba, offset, length)
             } catch (e: Exception) {
                 logger.debug(e) {
@@ -214,15 +199,6 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     public fun parseOrNull(cs: CharSequence): Double? {
         if (parserOptions.useFastDoubleParser) {
             try {
-                // Fixes RTL minus sign not being recognized
-                if (minusSignIsFormatSymbol && cs.startsWith(localDecimalFormatSymbols.minusSign)) {
-                    val newCs = cs.toString().replaceFirst(
-                        localDecimalFormatSymbols.minusSign,
-                        fallbackDecimalFormatSymbols.minusSign,
-                    )
-                    return parser.parseDouble(newCs)
-                }
-
                 return parser.parseDouble(cs)
             } catch (e: Exception) {
                 logger.debug(e) {
@@ -243,12 +219,6 @@ public class FastDoubleParser(private val parserOptions: ParserOptions) {
     public fun parseOrNull(ca: CharArray, offset: Int = 0, length: Int = ca.size): Double? {
         if (parserOptions.useFastDoubleParser) {
             try {
-                // Fixes RTL minus sign not being recognized.
-                if (minusSignIsFormatSymbol && ca.firstOrNull() == localDecimalFormatSymbols.minusSign) {
-                    val newCa = ca.copyOf()
-                    newCa[0] = fallbackDecimalFormatSymbols.minusSign
-                    return parser.parseDouble(newCa, offset, length)
-                }
                 return parser.parseDouble(ca, offset, length)
             } catch (e: Exception) {
                 logger.debug(e) {