Skip to content

Commit 1d2eb2e

Browse files
JolanrensenJolanrensen
authored
Merge pull request #903 from Kotlin/new-csv-implementation
New CSV implementation
2 parents 4a1f94f + a02b9e6 commit 1d2eb2e

File tree

76 files changed

+99231
-261
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+99231
-261
lines changed

‎.github/workflows/generated-sources-master.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
run: |
2727
git config --global user.name 'github-actions[bot]'
2828
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
29-
git add './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
29+
git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
3030
git diff --staged --quiet || git commit -m "Automated commit of generated code"
3131
git push
3232
env:

‎.github/workflows/generated-sources.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,18 @@ jobs:
3838
git config --global user.name "GitHub Actions"
3939
4040
- name: Run Gradle task
41-
run: ./gradlew :core:processKDocsMain korro
41+
run: ./gradlew processKDocsMain korro
4242

4343
- name: Check for changes in generated sources
4444
id: git-diff
45-
run: echo "changed=$(if git diff --quiet './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
45+
run: echo "changed=$(if git diff --quiet './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
4646

4747
- name: Commit and push if changes
4848
id: git-commit
4949
if: steps.git-diff.outputs.changed == 'true'
5050
run: |
5151
git checkout -b generated-sources/docs-update-${{ github.run_number }}
52-
git add './core/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
52+
git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/snippets' './docs/StardustDocs/topics'
5353
git commit -m "Update generated sources with recent changes"
5454
git push origin generated-sources/docs-update-${{ github.run_number }}
5555
echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

‎build.gradle.kts

+3
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,15 @@ dependencies {
5555
api(project(":dataframe-excel"))
5656
api(project(":dataframe-openapi"))
5757
api(project(":dataframe-jdbc"))
58+
// TODO enable when it leaves the experimental phase
59+
// api(project(":dataframe-csv"))
5860

5961
kover(project(":core"))
6062
kover(project(":dataframe-arrow"))
6163
kover(project(":dataframe-excel"))
6264
kover(project(":dataframe-openapi"))
6365
kover(project(":dataframe-jdbc"))
66+
kover(project(":dataframe-csv"))
6467
kover(project(":plugins:kotlin-dataframe"))
6568
}
6669

‎core/api/core.api

+45-14
Original file line numberDiff line numberDiff line change
@@ -3809,6 +3809,8 @@ public final class org/jetbrains/kotlinx/dataframe/api/ConvertKt {
38093809
public static final fun convert (Lorg/jetbrains/kotlinx/dataframe/DataFrame;[Lkotlin/reflect/KProperty;)Lorg/jetbrains/kotlinx/dataframe/api/Convert;
38103810
public static final fun convert (Lorg/jetbrains/kotlinx/dataframe/DataFrame;[Lorg/jetbrains/kotlinx/dataframe/columns/ColumnReference;)Lorg/jetbrains/kotlinx/dataframe/api/Convert;
38113811
public static final fun convertTo (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Lkotlin/reflect/KType;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3812+
public static final fun convertTo (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3813+
public static synthetic fun convertTo$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38123814
public static final fun convertToBigDecimal (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38133815
public static final fun convertToBigDecimalFromT (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38143816
public static final fun convertToBoolean (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
@@ -3817,13 +3819,13 @@ public final class org/jetbrains/kotlinx/dataframe/api/ConvertKt {
38173819
public static final fun convertToByteFromT (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38183820
public static final fun convertToDouble (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38193821
public static final fun convertToDoubleFromString (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3820-
public static final fun convertToDoubleFromString (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3822+
public static final fun convertToDoubleFromString (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Ljava/util/Set;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38213823
public static synthetic fun convertToDoubleFromString$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3822-
public static synthetic fun convertToDoubleFromString$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3824+
public static synthetic fun convertToDoubleFromString$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Ljava/util/Set;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38233825
public static final fun convertToDoubleFromStringNullable (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3824-
public static final fun convertToDoubleFromStringNullable (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Z)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3826+
public static final fun convertToDoubleFromStringNullable (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Ljava/util/Set;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38253827
public static synthetic fun convertToDoubleFromStringNullable$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3826-
public static synthetic fun convertToDoubleFromStringNullable$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
3828+
public static synthetic fun convertToDoubleFromStringNullable$default (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Ljava/util/Locale;Ljava/util/Set;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38273829
public static final fun convertToDoubleFromT (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38283830
public static final fun convertToFloat (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
38293831
public static final fun convertToFloatFromT (Lorg/jetbrains/kotlinx/dataframe/DataColumn;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
@@ -4813,9 +4815,14 @@ public final class org/jetbrains/kotlinx/dataframe/api/GenerateCodeKt {
48134815
public abstract interface class org/jetbrains/kotlinx/dataframe/api/GlobalParserOptions {
48144816
public abstract fun addDateTimePattern (Ljava/lang/String;)V
48154817
public abstract fun addNullString (Ljava/lang/String;)V
4818+
public abstract fun addSkipType (Lkotlin/reflect/KType;)V
48164819
public abstract fun getLocale ()Ljava/util/Locale;
4820+
public abstract fun getNulls ()Ljava/util/Set;
4821+
public abstract fun getSkipTypes ()Ljava/util/Set;
4822+
public abstract fun getUseFastDoubleParser ()Z
48174823
public abstract fun resetToDefault ()V
48184824
public abstract fun setLocale (Ljava/util/Locale;)V
4825+
public abstract fun setUseFastDoubleParser (Z)V
48194826
}
48204827

48214828
public abstract interface class org/jetbrains/kotlinx/dataframe/api/GroupBy : org/jetbrains/kotlinx/dataframe/api/Grouped {
@@ -6490,23 +6497,19 @@ public final class org/jetbrains/kotlinx/dataframe/api/ParserOptions {
64906497
public fun <init> ()V
64916498
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;)V
64926499
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
6493-
public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Z)V
6494-
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
6495-
public final fun component1 ()Ljava/util/Locale;
6496-
public final fun component2 ()Ljava/time/format/DateTimeFormatter;
6497-
public final fun component3 ()Ljava/lang/String;
6498-
public final fun component4 ()Ljava/util/Set;
6499-
public final fun component5 ()Z
6500+
public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;)V
6501+
public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
65006502
public final synthetic fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
6501-
public final fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Z)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
6503+
public final fun copy (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
65026504
public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
6503-
public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
6505+
public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
65046506
public fun equals (Ljava/lang/Object;)Z
65056507
public final fun getDateTimeFormatter ()Ljava/time/format/DateTimeFormatter;
65066508
public final fun getDateTimePattern ()Ljava/lang/String;
65076509
public final fun getLocale ()Ljava/util/Locale;
65086510
public final fun getNullStrings ()Ljava/util/Set;
6509-
public final fun getUseFastDoubleParser ()Z
6511+
public final fun getSkipTypes ()Ljava/util/Set;
6512+
public final fun getUseFastDoubleParser ()Ljava/lang/Boolean;
65106513
public fun hashCode ()I
65116514
public fun toString ()Ljava/lang/String;
65126515
}
@@ -9948,6 +9951,16 @@ public final class org/jetbrains/kotlinx/dataframe/impl/ColumnAccessTrackerKt {
99489951
public static final fun trackColumnAccess (Lkotlin/jvm/functions/Function0;)Ljava/util/List;
99499952
}
99509953

9954+
public final class org/jetbrains/kotlinx/dataframe/impl/ColumnNameGenerator {
9955+
public fun <init> ()V
9956+
public fun <init> (Ljava/util/List;)V
9957+
public synthetic fun <init> (Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
9958+
public final fun addIfAbsent (Ljava/lang/String;)V
9959+
public final fun addUnique (Ljava/lang/String;)Ljava/lang/String;
9960+
public final fun contains (Ljava/lang/String;)Z
9961+
public final fun getNames ()Ljava/util/List;
9962+
}
9963+
99519964
public final class org/jetbrains/kotlinx/dataframe/impl/DataFrameSize {
99529965
public fun <init> (II)V
99539966
public final fun component1 ()I
@@ -10211,7 +10224,9 @@ public final class org/jetbrains/kotlinx/dataframe/impl/columns/UtilsKt {
1021110224
}
1021210225

1021310226
public final class org/jetbrains/kotlinx/dataframe/impl/io/FastDoubleParser {
10227+
public fun <init> ()V
1021410228
public fun <init> (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;)V
10229+
public synthetic fun <init> (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
1021510230
public final fun parseOrNull (Ljava/lang/CharSequence;)Ljava/lang/Double;
1021610231
public final fun parseOrNull ([BIILjava/nio/charset/Charset;)Ljava/lang/Double;
1021710232
public final fun parseOrNull ([CII)Ljava/lang/Double;
@@ -10273,23 +10288,38 @@ public final class org/jetbrains/kotlinx/dataframe/io/CSVType : java/lang/Enum {
1027310288
public final class org/jetbrains/kotlinx/dataframe/io/ColType : java/lang/Enum {
1027410289
public static final field BigDecimal Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1027510290
public static final field Boolean Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10291+
public static final field Char Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10292+
public static final field Companion Lorg/jetbrains/kotlinx/dataframe/io/ColType$Companion;
10293+
public static final field DEFAULT Ljava/lang/String;
1027610294
public static final field Double Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10295+
public static final field Duration Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10296+
public static final field Instant Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1027710297
public static final field Int Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10298+
public static final field JsonArray Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10299+
public static final field JsonObject Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1027810300
public static final field LocalDate Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1027910301
public static final field LocalDateTime Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028010302
public static final field LocalTime Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028110303
public static final field Long Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028210304
public static final field String Lorg/jetbrains/kotlinx/dataframe/io/ColType;
10305+
public static final field Url Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028310306
public static fun getEntries ()Lkotlin/enums/EnumEntries;
1028410307
public static fun valueOf (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028510308
public static fun values ()[Lorg/jetbrains/kotlinx/dataframe/io/ColType;
1028610309
}
1028710310

10311+
public final class org/jetbrains/kotlinx/dataframe/io/ColType$Companion {
10312+
}
10313+
1028810314
public final class org/jetbrains/kotlinx/dataframe/io/CommonKt {
1028910315
public static final fun asFileOrNull (Ljava/net/URL;)Ljava/io/File;
10316+
public static final fun asUrl (Ljava/lang/String;)Ljava/net/URL;
10317+
public static final fun catchHttpResponse (Ljava/net/URL;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
1029010318
public static final fun isFile (Ljava/net/URL;)Z
1029110319
public static final fun isProtocolSupported (Ljava/net/URL;)Z
1029210320
public static final fun isURL (Ljava/lang/String;)Z
10321+
public static final fun isUrl (Ljava/lang/String;)Z
10322+
public static final fun skippingBomCharacters (Ljava/io/InputStream;)Ljava/io/InputStream;
1029310323
public static final fun toDataFrame (Ljava/util/List;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
1029410324
public static synthetic fun toDataFrame$default (Ljava/util/List;ZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
1029510325
public static final fun urlAsFile (Ljava/net/URL;)Ljava/io/File;
@@ -10315,6 +10345,7 @@ public final class org/jetbrains/kotlinx/dataframe/io/CsvKt {
1031510345
public static synthetic fun readDelimStr$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/String;CLjava/util/Map;ILjava/lang/Integer;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
1031610346
public static final fun toCsv (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/apache/commons/csv/CSVFormat;)Ljava/lang/String;
1031710347
public static synthetic fun toCsv$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/apache/commons/csv/CSVFormat;ILjava/lang/Object;)Ljava/lang/String;
10348+
public static final fun toKType (Lorg/jetbrains/kotlinx/dataframe/io/ColType;)Lkotlin/reflect/KType;
1031810349
public static final fun toType (Lorg/jetbrains/kotlinx/dataframe/io/ColType;)Lkotlin/reflect/KClass;
1031910350
public static final fun writeCSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Ljava/io/File;Lorg/apache/commons/csv/CSVFormat;)V
1032010351
public static final fun writeCSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Ljava/lang/Appendable;Lorg/apache/commons/csv/CSVFormat;)V

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public enum class DataSchemaVisibility {
4747
EXPLICIT_PUBLIC,
4848
}
4949

50+
// TODO add more options
5051
public annotation class CsvOptions(public val delimiter: Char)
5152

5253
/**

0 commit comments

Comments
 (0)