From 7717779ebff3d5900c3adcd06bc860949d543f97 Mon Sep 17 00:00:00 2001 From: Kunni Date: Thu, 6 Feb 2025 14:07:38 +0800 Subject: [PATCH] [FLINK-37231][docs] Add documentation for CDC Source metrics This closes #3897 --- .../docs/connectors/flink-sources/db2-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/mongodb-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/mysql-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/oceanbase-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/oracle-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/postgres-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/sqlserver-cdc.md | 20 ++++++++++++++++++ .../connectors/pipeline-connectors/mysql.md | 20 ++++++++++++++++++ .../docs/connectors/flink-sources/db2-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/mongodb-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/mysql-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/oceanbase-cdc.md | 21 +++++++++++++++++++ .../connectors/flink-sources/oracle-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/postgres-cdc.md | 20 ++++++++++++++++++ .../connectors/flink-sources/sqlserver-cdc.md | 20 ++++++++++++++++++ .../connectors/pipeline-connectors/mysql.md | 20 ++++++++++++++++++ 16 files changed, 321 insertions(+) diff --git a/docs/content.zh/docs/connectors/flink-sources/db2-cdc.md b/docs/content.zh/docs/connectors/flink-sources/db2-cdc.md index 673a4afc140..e6dad221693 100644 --- a/docs/content.zh/docs/connectors/flink-sources/db2-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/db2-cdc.md @@ -354,6 +354,26 @@ public class Db2SourceExample { } ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 DB2，Group 的名称会类似于 `test_database.test_schema.test_table`。 + The DB2 CDC incremental connector (since 3.1.0) can be used as the following shows: ```java import org.apache.flink.api.common.eventtime.WatermarkStrategy; diff --git a/docs/content.zh/docs/connectors/flink-sources/mongodb-cdc.md b/docs/content.zh/docs/connectors/flink-sources/mongodb-cdc.md index 9491de62212..14167a47e22 100644 --- a/docs/content.zh/docs/connectors/flink-sources/mongodb-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/mongodb-cdc.md @@ -488,6 +488,26 @@ public class MongoDBIncrementalSourceExample { - 如果使用数据库正则表达式，则需要 `readAnyDatabase` 角色。 - 增量快照功能仅支持 MongoDB 4.0 之后的版本。 +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 MongoDB，这里的 `namespace` 会被设置成默认值 ""，也就是一个空字符串，Group 名称的格式会类似于 `test_database.test_table`。 + ### 完整的 Changelog MongoDB 6.0 以及更高的版本支持发送变更流事件，其中包含文档的更新前和更新后的内容（或者说数据的前后镜像）。 diff --git a/docs/content.zh/docs/connectors/flink-sources/mysql-cdc.md b/docs/content.zh/docs/connectors/flink-sources/mysql-cdc.md index a2a6f6ad70b..9cac2ad7775 100644 --- a/docs/content.zh/docs/connectors/flink-sources/mysql-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/mysql-cdc.md @@ -776,6 +776,26 @@ $ ./bin/flink run \ * 如果指定的列不存在更新操作，此时可以保证 Exactly once 语义。 * 如果指定的列存在更新操作，此时只能保证 At least once 语义。但可以结合下游，通过指定下游主键，结合幂等性操作来保证数据的正确性。 +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 MySQL，这里的 `namespace` 会被设置成默认值 ""，也就是一个空字符串，Group 名称的格式会类似于 `test_database.test_table`。 + 数据类型映射 ---------------- diff --git a/docs/content.zh/docs/connectors/flink-sources/oceanbase-cdc.md b/docs/content.zh/docs/connectors/flink-sources/oceanbase-cdc.md index e61e3636fc2..70d8e6a1229 100644 --- a/docs/content.zh/docs/connectors/flink-sources/oceanbase-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/oceanbase-cdc.md @@ -569,6 +569,26 @@ public class OceanBaseSourceExample { } ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 OceanBase，这里的 `namespace` 会被设置成默认值 ""，也就是一个空字符串，Group 名称的格式会类似于 `test_database.test_table`。 + 数据类型映射 ---------------- diff --git a/docs/content.zh/docs/connectors/flink-sources/oracle-cdc.md b/docs/content.zh/docs/connectors/flink-sources/oracle-cdc.md index 22dc470459d..5ba089d99b5 100644 --- a/docs/content.zh/docs/connectors/flink-sources/oracle-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/oracle-cdc.md @@ -614,6 +614,26 @@ public class OracleSourceExample { } ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 Oracle，Group 的名称会类似于 `test_database.test_schema.test_table`。 + Data Type Mapping ----------------
diff --git a/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md b/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md index d1a8a0a78f4..6ff470050c9 100644 --- a/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md @@ -538,6 +538,26 @@ public class PostgreSQLSourceExample { } ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 PostgreSQL，Group 的名称会类似于 `test_database.test_schema.test_table`。 + Data Type Mapping ---------------- diff --git a/docs/content.zh/docs/connectors/flink-sources/sqlserver-cdc.md b/docs/content.zh/docs/connectors/flink-sources/sqlserver-cdc.md index 9a6f8a9ddfd..3f342456b1b 100644 --- a/docs/content.zh/docs/connectors/flink-sources/sqlserver-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/sqlserver-cdc.md @@ -409,6 +409,26 @@ public class SqlServerIncrementalSourceExample { } ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 SQLServer，Group 的名称会类似于 `test_database.test_schema.test_table`。 + Data Type Mapping ---------------- diff --git a/docs/content.zh/docs/connectors/pipeline-connectors/mysql.md b/docs/content.zh/docs/connectors/pipeline-connectors/mysql.md index afd1897167b..8afeddf5247 100644 --- a/docs/content.zh/docs/connectors/pipeline-connectors/mysql.md +++ b/docs/content.zh/docs/connectors/pipeline-connectors/mysql.md @@ -343,6 +343,26 @@ source: # ... ``` +### 可用的指标 + +指标系统能够帮助了解分片分发的进展， 下面列举出了支持的 Flink 指标 [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|----------------| +| namespace.schema.table | isSnapshotting | Gauge | 表是否在快照读取阶段 | +| namespace.schema.table | isStreamReading | Gauge | 表是否在增量读取阶段 | +| namespace.schema.table | numTablesSnapshotted | Gauge | 已经被快照读取完成的表的数量 | +| namespace.schema.table | numTablesRemaining | Gauge | 还没有被快照读取的表的数据 | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | 正在处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | 还没有被处理的分片的数量 | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | 已经处理完成的分片的数据 | +| namespace.schema.table | snapshotStartTime | Gauge | 快照读取阶段开始的时间 | +| namespace.schema.table | snapshotEndTime | Gauge | 快照读取阶段结束的时间 | + +注意: +1. Group 名称是 `namespace.schema.table`，这里的 `namespace` 是实际的数据库名称， `schema` 是实际的 schema 名称， `table` 是实际的表名称。 +2. 对于 MySQL，这里的 `namespace` 会被设置成默认值 ""，也就是一个空字符串，Group 名称的格式会类似于 `test_database.test_table`。 + ## 数据类型映射
diff --git a/docs/content/docs/connectors/flink-sources/db2-cdc.md b/docs/content/docs/connectors/flink-sources/db2-cdc.md index b7ebc93aa9d..700c169faba 100644 --- a/docs/content/docs/connectors/flink-sources/db2-cdc.md +++ b/docs/content/docs/connectors/flink-sources/db2-cdc.md @@ -394,6 +394,26 @@ public class Db2ParallelSourceExample { } ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For DB2, the group name will be like `test_database.test_schema.test_table`. + Data Type Mapping ---------------- diff --git a/docs/content/docs/connectors/flink-sources/mongodb-cdc.md b/docs/content/docs/connectors/flink-sources/mongodb-cdc.md index feb27b1da5e..12f35aa76e4 100644 --- a/docs/content/docs/connectors/flink-sources/mongodb-cdc.md +++ b/docs/content/docs/connectors/flink-sources/mongodb-cdc.md @@ -563,6 +563,26 @@ CREATE TABLE mongodb_source (...) WITH ( ) ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For MongoDB, the `namespace` will be set to the default value "", and the group name will be like `test_database.test_table`. + Data Type Mapping ---------------- [BSON](https://docs.mongodb.com/manual/reference/bson-types/) short for **Binary JSON** is a binary-encoded serialization of JSON-like format used to store documents and make remote procedure calls in MongoDB. diff --git a/docs/content/docs/connectors/flink-sources/mysql-cdc.md b/docs/content/docs/connectors/flink-sources/mysql-cdc.md index 4bd22a0a162..12cbb6fde11 100644 --- a/docs/content/docs/connectors/flink-sources/mysql-cdc.md +++ b/docs/content/docs/connectors/flink-sources/mysql-cdc.md @@ -852,6 +852,26 @@ CREATE TABLE products ( `binary_data` field in the database is of type VARBINARY(N). In some scenarios, we need to convert binary data to base64 encoded string data. This feature can be enabled by adding the parameter 'debezium.binary.handling.mode'='base64', By adding this parameter, we can map the binary field type to 'STRING' in Flink SQL, thereby obtaining base64 encoded string data. +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For MySQL, the `namespace` will be set to the default value "", and the group name will be like `test_database.test_table`. + Data Type Mapping ---------------- diff --git a/docs/content/docs/connectors/flink-sources/oceanbase-cdc.md b/docs/content/docs/connectors/flink-sources/oceanbase-cdc.md index 3b03009cc00..bc390ee369d 100644 --- a/docs/content/docs/connectors/flink-sources/oceanbase-cdc.md +++ b/docs/content/docs/connectors/flink-sources/oceanbase-cdc.md @@ -238,6 +238,7 @@ Flink SQL> CREATE TABLE orders ( You can also try the quickstart tutorial that sync data from OceanBase to Elasticsearch, please refer [Flink CDC Tutorial]({{< ref "docs/connectors/flink-sources/tutorials/oceanbase-tutorial" >}}) for more information. + Connector Options ---------------- @@ -566,6 +567,26 @@ public class OceanBaseSourceExample { } ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For OceanBase, the `namespace` will be set to the default value "", and the group name will be like `test_database.test_table`. + Data Type Mapping ---------------- diff --git a/docs/content/docs/connectors/flink-sources/oracle-cdc.md b/docs/content/docs/connectors/flink-sources/oracle-cdc.md index 87108d42757..5513a4df24e 100644 --- a/docs/content/docs/connectors/flink-sources/oracle-cdc.md +++ b/docs/content/docs/connectors/flink-sources/oracle-cdc.md @@ -615,6 +615,26 @@ public class OracleSourceExample { } ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For Oracle, the group name will be like `test_database.test_schema.test_table`. + Data Type Mapping ----------------
diff --git a/docs/content/docs/connectors/flink-sources/postgres-cdc.md b/docs/content/docs/connectors/flink-sources/postgres-cdc.md index be3d826d0d8..ea8b347a02b 100644 --- a/docs/content/docs/connectors/flink-sources/postgres-cdc.md +++ b/docs/content/docs/connectors/flink-sources/postgres-cdc.md @@ -539,6 +539,26 @@ public class PostgreSQLSourceExample { } ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For PostgreSQL, the group name will be like `test_database.test_schema.test_table`. + ## Data Type Mapping
diff --git a/docs/content/docs/connectors/flink-sources/sqlserver-cdc.md b/docs/content/docs/connectors/flink-sources/sqlserver-cdc.md index 9a6f8a9ddfd..d519413dc7c 100644 --- a/docs/content/docs/connectors/flink-sources/sqlserver-cdc.md +++ b/docs/content/docs/connectors/flink-sources/sqlserver-cdc.md @@ -409,6 +409,26 @@ public class SqlServerIncrementalSourceExample { } ``` +### Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For SQLServer, the group name will be like `test_database.test_schema.test_table`. + Data Type Mapping ---------------- diff --git a/docs/content/docs/connectors/pipeline-connectors/mysql.md b/docs/content/docs/connectors/pipeline-connectors/mysql.md index b05e67921f4..0d1f07a528f 100644 --- a/docs/content/docs/connectors/pipeline-connectors/mysql.md +++ b/docs/content/docs/connectors/pipeline-connectors/mysql.md @@ -366,6 +366,26 @@ source: # ... ``` +## Available Source metrics + +Metrics can help understand the progress of assignments, and the following are the supported [Flink metrics](https://nightlies.apache.org/flink/flink-docs-master/docs/ops/metrics/): + +| Group | Name | Type | Description | +|------------------------|----------------------------|-------|-----------------------------------------------------| +| namespace.schema.table | isSnapshotting | Gauge | Weather the table is snapshotting or not | +| namespace.schema.table | isStreamReading | Gauge | Weather the table is stream reading or not | +| namespace.schema.table | numTablesSnapshotted | Gauge | The number of tables that have been snapshotted | +| namespace.schema.table | numTablesRemaining | Gauge | The number of tables that have not been snapshotted | +| namespace.schema.table | numSnapshotSplitsProcessed | Gauge | The number of splits that is being processed | +| namespace.schema.table | numSnapshotSplitsRemaining | Gauge | The number of splits that have not been processed | +| namespace.schema.table | numSnapshotSplitsFinished | Gauge | The number of splits that have been processed | +| namespace.schema.table | snapshotStartTime | Gauge | The time when the snapshot started | +| namespace.schema.table | snapshotEndTime | Gauge | The time when the snapshot ended | + +Notice: +1. The group name is `namespace.schema.table`, where `namespace` is the actual database name, `schema` is the actual schema name, and `table` is the actual table name. +2. For MySQL, the `namespace` will be set to the default value "", and the group name will be like `test_database.test_table`. + ## Data Type Mapping