From fe616ca0a8dc8e38c086a8d7b5ed7b272203f62a Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 3 Nov 2026 15:38:50 +0800 Subject: [PATCH] docs: clarify continuous load experimental versions --- .../continuous-load-mysql-database.md | 20 ++++++++----- .../continuous-load-mysql-table.md | 12 +++++--- .../continuous-load-oceanbase-database.md | 12 ++++---- .../streaming-job/continuous-load-overview.md | 18 ++++++----- .../continuous-load-postgresql-database.md | 22 +++++++++----- .../continuous-load-postgresql-table.md | 10 +++++-- .../continuous-load-s3-notification.md | 6 ++++ .../continuous-load-s3-one-time.md | 6 ++++ .../continuous-load-s3-ordered.md | 6 ++++ .../streaming-job/continuous-load-s3.md | 8 ++++- .../streaming-job/data-type-mapping-mysql.md | 14 +++++---- .../data-type-mapping-oceanbase.md | 10 +++---- .../data-type-mapping-postgresql.md | 6 ++++ .../prerequisites/amazon-aurora-mysql.md | 6 ++++ .../prerequisites/amazon-aurora-postgresql.md | 27 +++++++---------- .../prerequisites/amazon-rds-mysql.md | 6 ++++ .../prerequisites/amazon-rds-postgresql.md | 27 +++++++---------- .../streaming-job/schema-change-mysql.md | 12 ++++---- .../streaming-job/schema-change-oceanbase.md | 12 ++++---- .../streaming-job/schema-change-postgresql.md | 18 +++++------ .../current.json | 18 +++++------ .../version-4.x.json | 18 +++++------ .../current.json | 18 +++++------ .../version-4.x.json | 18 +++++------ .../current.json | 18 +++++------ .../version-4.x.json | 18 +++++------ .../current.json | 18 +++++------ .../continuous-load-mysql-database.md | 20 ++++++++----- .../continuous-load-mysql-table.md | 12 +++++--- .../continuous-load-oceanbase-database.md | 12 ++++---- .../streaming-job/continuous-load-overview.md | 18 ++++++----- .../continuous-load-postgresql-database.md | 22 +++++++++----- .../continuous-load-postgresql-table.md | 10 +++++-- .../continuous-load-s3-notification.md | 6 ++++ .../continuous-load-s3-one-time.md | 6 ++++ .../continuous-load-s3-ordered.md | 6 ++++ .../streaming-job/continuous-load-s3.md | 8 ++++- .../streaming-job/data-type-mapping-mysql.md | 14 +++++---- .../data-type-mapping-oceanbase.md | 10 +++---- .../data-type-mapping-postgresql.md | 6 ++++ .../prerequisites/amazon-aurora-mysql.md | 6 ++++ .../prerequisites/amazon-aurora-postgresql.md | 27 +++++++---------- .../prerequisites/amazon-rds-mysql.md | 6 ++++ .../prerequisites/amazon-rds-postgresql.md | 27 +++++++---------- .../streaming-job/schema-change-mysql.md | 12 ++++---- .../streaming-job/schema-change-oceanbase.md | 12 ++++---- .../streaming-job/schema-change-postgresql.md | 14 ++++----- .../version-4.x.json | 18 +++++------ .../continuous-load-mysql-database.md | 8 ++++- .../continuous-load-mysql-table.md | 10 +++++-- .../continuous-load-oceanbase-database.md | 12 ++++---- .../streaming-job/continuous-load-overview.md | 18 ++++++----- .../continuous-load-postgresql-database.md | 8 ++++- .../continuous-load-postgresql-table.md | 8 ++++- .../continuous-load-s3-notification.md | 6 ++++ .../continuous-load-s3-one-time.md | 6 ++++ .../continuous-load-s3-ordered.md | 6 ++++ .../streaming-job/continuous-load-s3.md | 8 ++++- .../streaming-job/data-type-mapping-mysql.md | 14 +++++---- .../data-type-mapping-oceanbase.md | 10 +++---- .../data-type-mapping-postgresql.md | 6 ++++ .../prerequisites/amazon-aurora-mysql.md | 6 ++++ .../prerequisites/amazon-aurora-postgresql.md | 6 ++++ .../prerequisites/amazon-rds-mysql.md | 6 ++++ .../prerequisites/amazon-rds-postgresql.md | 6 ++++ .../streaming-job/schema-change-mysql.md | 12 ++++---- .../streaming-job/schema-change-oceanbase.md | 12 ++++---- .../streaming-job/schema-change-postgresql.md | 16 +++++----- sidebars.ts | 6 ++-- .../continuous-load-mysql-database.md | 8 ++++- .../continuous-load-mysql-table.md | 10 +++++-- .../continuous-load-oceanbase-database.md | 12 ++++---- .../streaming-job/continuous-load-overview.md | 30 ++++++++++--------- .../continuous-load-postgresql-database.md | 8 ++++- .../continuous-load-postgresql-table.md | 8 ++++- .../continuous-load-s3-notification.md | 6 ++++ .../continuous-load-s3-one-time.md | 6 ++++ .../continuous-load-s3-ordered.md | 6 ++++ .../streaming-job/continuous-load-s3.md | 8 ++++- .../streaming-job/data-type-mapping-mysql.md | 14 +++++---- .../data-type-mapping-oceanbase.md | 10 +++---- .../data-type-mapping-postgresql.md | 6 ++++ .../prerequisites/amazon-aurora-mysql.md | 6 ++++ .../prerequisites/amazon-aurora-postgresql.md | 6 ++++ .../prerequisites/amazon-rds-mysql.md | 6 ++++ .../prerequisites/amazon-rds-postgresql.md | 6 ++++ .../streaming-job/schema-change-mysql.md | 12 ++++---- .../streaming-job/schema-change-oceanbase.md | 12 ++++---- .../streaming-job/schema-change-postgresql.md | 18 +++++------ versioned_sidebars/version-4.x-sidebars.json | 6 ++-- 90 files changed, 671 insertions(+), 379 deletions(-) diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md index 7fbbebf6142329..106a09607f3bbd 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md @@ -17,6 +17,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -145,14 +151,14 @@ The MySQL source (`FROM MYSQL`) supports the following parameters. Connection in | `database` | Yes | - | MySQL database name. | | `include_tables` | No | - | Names of the tables to sync, separated by commas. If not set, all tables in the database are synced. | | `exclude_tables` | No | - | Names of the tables not to sync, separated by commas. This parameter takes effect only when `include_tables` is not set. | -| `table..target_table` | No | Source table name | Supported since version 4.1.0. Sets the Doris target table name for a source table. `` is the source table name. | -| `table..exclude_columns` | No | - | Supported since version 4.1.0. Specifies source columns not to sync, separated by commas. The columns must exist and cannot include primary key columns. | -| `offset` | No | `latest` | Startup offset. `initial`: full and incremental sync; `earliest`: start from the earliest available Binlog offset; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"file":"binlog.000001","pos":"154"}` or `{"gtids":""}`. Since version 4.1.0, `snapshot` is also supported for full sync only. | -| `snapshot_split_size` | No | `40960` (`8096` before 4.1.4) | Split size in rows. During full sync, a table is divided into multiple splits. Must be a positive integer. | +| `table..target_table` | No | Source table name | Sets the Doris target table name for a source table. `` is the source table name. | +| `table..exclude_columns` | No | - | Specifies source columns not to sync, separated by commas. The columns must exist and cannot include primary key columns. | +| `offset` | No | `latest` | Startup offset. `initial`: full and incremental sync; `earliest`: start from the earliest available Binlog offset; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"file":"binlog.000001","pos":"154"}` or `{"gtids":""}`. `snapshot` is also supported for full sync only. | +| `snapshot_split_size` | No | `40960` | Split size in rows. During full sync, a table is divided into multiple splits. Must be a positive integer. | | `snapshot_parallelism` | No | `1` | Parallelism of the full-sync phase, that is, the maximum number of splits scheduled by a Task at one time. Must be a positive integer. | -| `server_id` | No | Automatically generated | Supported since version 4.1.0. Server ID of the MySQL CDC reader. Supports a single value, such as `5400`, or a closed range, such as `5400-5408`. The range width must be greater than or equal to `snapshot_parallelism`. | -| `ssl_mode` | No | `disable` | Supported since version 4.1.0. SSL mode. Valid values are `disable`, `require`, and `verify-ca`. | -| `ssl_rootcert` | Conditionally required | - | Supported since version 4.1.0. CA certificate file in the format `FILE:`. Required when `ssl_mode` is `verify-ca`. Upload the file first using [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md). | +| `server_id` | No | Automatically generated | Server ID of the MySQL CDC reader. Supports a single value, such as `5400`, or a closed range, such as `5400-5408`. The range width must be greater than or equal to `snapshot_parallelism`. | +| `ssl_mode` | No | `disable` | SSL mode. Valid values are `disable`, `require`, and `verify-ca`. | +| `ssl_rootcert` | Conditionally required | - | CA certificate file in the format `FILE:`. Required when `ssl_mode` is `verify-ca`. Upload the file first using [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md). | If both `include_tables` and `exclude_tables` are set, `include_tables` takes precedence. The following `FROM MYSQL` parameter fragment syncs `orders` and `customers`, writes `orders` to `ods_orders`, excludes the non-key column `internal_note`, and enables CA verification: diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md index b234082e2116d7..beca83534df4d2 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md @@ -17,13 +17,17 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + SQL Mapping Sync is implemented through Job + [CDC Stream TVF](../../../../sql-manual/sql-functions/table-valued-functions/cdc-stream.md), targeting an **existing** Doris table (`INSERT INTO tbl SELECT * FROM cdc_stream(...)`). With the expressive power of Doris SQL, you can perform column mapping, filtering, and data transformation in the sync pipeline. This is suitable for real-time sync scenarios that require data processing. -> SQL Mapping Sync is supported since version 4.1.0. - By integrating the read capability of [Flink CDC](https://github.com/apache/flink-cdc), Doris reads the change log (Binlog) from MySQL and completes **full + incremental** sync from source table to target table. If you want Doris to automatically create downstream tables and sync a group of tables on a per-database basis, see [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md). ### Applicable Scenarios @@ -36,7 +40,7 @@ By integrating the read capability of [Flink CDC](https://github.com/apache/flin | Item | Description | | ------ | ---- | -| Doris version | 4.1.0 and above | +| Doris version | 4.1.5 and above | | Table type | The upstream source table must have a primary key; the Doris target table must use the Unique Key model | | User privileges | `Load` privilege is required | | MySQL configuration | Binlog must be enabled on the MySQL side. See the [Configuration Guide](./continuous-load-overview.md#supported-data-sources-and-sync-modes) | @@ -95,7 +99,7 @@ CDC Stream TVF supports the following MySQL data source parameters. | `database` | Yes | - | MySQL database name. | | `table` | Yes | - | Name of the table to sync. Each SQL Mapping job supports one source table. | | `offset` | Yes | - | Startup offset. `initial`: full and incremental sync; `snapshot`: full sync only; `earliest`: start from the earliest available Binlog offset; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"file":"binlog.000001","pos":"154"}` or `{"gtids":""}`. | -| `snapshot_split_size` | No | `40960` (`8096` before 4.1.4) | Split size in rows. During full sync, the table is divided into multiple splits. Must be a positive integer. | +| `snapshot_split_size` | No | `40960` | Split size in rows. During full sync, the table is divided into multiple splits. Must be a positive integer. | | `snapshot_parallelism` | No | `1` | Parallelism of the full-sync phase, that is, the maximum number of splits scheduled by a Task at one time. Must be a positive integer. | | `skip_snapshot_backfill` | No | `false` | Whether to skip Binlog backfill during the snapshot. When set to `true`, at-least-once semantics are used. | | `server_id` | No | Automatically generated | Server ID of the MySQL CDC reader. Supports a single value, such as `5400`, or a closed range, such as `5400-5408`. The range width must be greater than or equal to `snapshot_parallelism`. | diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md index 20a29ab267bb92..eaf23f02783a09 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md @@ -16,16 +16,18 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + OceanBase CDC with Auto Table Creation uses `FROM OCEANBASE (...) TO DATABASE (...)`. Doris reads full and incremental data through the OBBinlog service and automatically creates Doris Unique Key tables from the upstream schemas. Use `include_tables` to sync one table, multiple tables, or all tables in the database. -:::caution Experimental feature - -OceanBase CDC continuous load is available as an experimental feature starting from Doris 4.1.4. It currently supports only OceanBase MySQL compatibility mode and Auto Table Creation Sync. Oracle compatibility mode and SQL Mapping Sync are not supported. - -::: +OceanBase CDC continuous load currently supports only OceanBase MySQL compatibility mode and Auto Table Creation Sync. Oracle compatibility mode and SQL Mapping Sync are not supported. ### Applicable Scenarios diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-overview.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-overview.md index d3c6c53bf26669..c2fdd0f8cd6ea6 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-overview.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-overview.md @@ -20,15 +20,17 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + Doris supports continuously loading data from multiple data sources into Doris tables through **Streaming Job**. After a Job is submitted, Doris keeps the load job running, reading data from the source in real time and writing it into the Doris table. -:::tip -This feature is supported starting from version 4.1.0. -::: - This document helps you answer the following questions: - Which data sources and sync modes does continuous load support? @@ -39,13 +41,13 @@ This document helps you answer the following questions: ## Supported Data Sources and Sync Modes -Continuous load supports the following data sources and sync modes: +Continuous load supports the following data sources and sync modes. S3 notification and one-time ingestion are supported starting with Doris 4.1.6: | Data Source | Supported Versions | SQL Mapping Sync | Auto Table Creation Sync | Configuration Guide | | :---------- | :----------------- | :---------------------------------------------------------------- | :---------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | | MySQL | 5.6, 5.7, 8.0.x | [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md) | [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md) | [Amazon RDS MySQL](./prerequisites/amazon-rds-mysql.md) · [Amazon Aurora MySQL](./prerequisites/amazon-aurora-mysql.md) | | PostgreSQL | 14, 15, 16, 17 | [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md) | [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md) | [Amazon RDS PostgreSQL](./prerequisites/amazon-rds-postgresql.md) · [Amazon Aurora PostgreSQL](./prerequisites/amazon-aurora-postgresql.md) | -| OceanBase | MySQL compatibility mode | - | [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md) (since version 4.1.4) | - | +| OceanBase | MySQL compatibility mode | - | [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md) | - | | S3 | - | [S3 Continuous Load](./continuous-load-s3.md) | - | - | For how upstream column types map to Doris types, see Data Type Mapping for [MySQL](./data-type-mapping-mysql.md), [PostgreSQL](./data-type-mapping-postgresql.md), and [OceanBase](./data-type-mapping-oceanbase.md). @@ -142,8 +144,8 @@ Result columns: | LoadStatistic | Job statistics | | ErrorMsg | Error message of the Job | | JobRuntimeMsg | Runtime hints of the Job | -| LagBytes | Number of backlog bytes in the source log (MySQL / OceanBase binlog or PostgreSQL WAL). `-1` means the value is currently unavailable, for example for an S3 data source or during the full snapshot phase. **Since version 4.1.4**, this column replaces the former `Lag` column (in seconds) and is reported in bytes | -| LastSourceEventTimestamp | Timestamp (in Unix seconds) of the latest source event recorded in the committed offset. Empty when unavailable. **Added in version 4.1.4** | +| LagBytes | Number of backlog bytes in the source log (MySQL / OceanBase binlog or PostgreSQL WAL). `-1` means the value is currently unavailable, for example for an S3 data source or during the full snapshot phase. | +| LastSourceEventTimestamp | Timestamp (in Unix seconds) of the latest source event recorded in the committed offset. Empty when unavailable. | | LastTaskSuccessTime | Time when the most recent Task completed successfully | ### View Task Status diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md index 7c3f57bdeb1a36..9f8d379708f35a 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md @@ -19,6 +19,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -96,15 +102,15 @@ PostgreSQL source parameters configure the JDBC connection, sync scope, and full | `schema` | Yes | - | Schema name. | | `include_tables` | No | - | Names of the tables to sync, separated by commas. If not set, all tables in the Schema are synced. | | `exclude_tables` | No | - | Names of the tables not to sync, separated by commas. This parameter takes effect only when `include_tables` is not set. | -| `table..target_table` | No | Source table name | Supported since version 4.1.0. Sets the Doris target table name for a source table. `` is the source table name. | -| `table..exclude_columns` | No | - | Supported since version 4.1.0. Specifies source columns not to sync, separated by commas. The columns must exist and cannot include primary key columns. | -| `offset` | No | `latest` | Startup offset. `initial`: full and incremental sync; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"lsn":"12345678"}`. Since version 4.1.0, `snapshot` is also supported for full sync only. PostgreSQL does not support `earliest`. | -| `snapshot_split_size` | No | `40960` (`8096` before 4.1.4) | Split size in rows. During full sync, a table is divided into multiple splits. Must be a positive integer. | +| `table..target_table` | No | Source table name | Sets the Doris target table name for a source table. `` is the source table name. | +| `table..exclude_columns` | No | - | Specifies source columns not to sync, separated by commas. The columns must exist and cannot include primary key columns. | +| `offset` | No | `latest` | Startup offset. `initial`: full and incremental sync; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"lsn":"12345678"}`. `snapshot` is also supported for full sync only. PostgreSQL does not support `earliest`. | +| `snapshot_split_size` | No | `40960` | Split size in rows. During full sync, a table is divided into multiple splits. Must be a positive integer. | | `snapshot_parallelism` | No | `1` | Parallelism of the full-sync phase, that is, the maximum number of splits scheduled by a Task at one time. Must be a positive integer. | -| `slot_name` | No | `doris_cdc_` | Supported since version 4.1.0. Logical replication slot name. The name can contain only lowercase letters, digits, and underscores, cannot start with a digit, and is limited to 63 characters. A custom slot must be created in advance and is not deleted by Doris. | -| `publication_name` | No | `doris_pub_` | Supported since version 4.1.0. Publication name. The naming rules are the same as for `slot_name`. A custom publication must be created in advance, include all synced tables, and is not deleted by Doris. | -| `ssl_mode` | No | `disable` | Supported since version 4.1.0. SSL mode. Valid values are `disable`, `require`, and `verify-ca`. | -| `ssl_rootcert` | Conditionally required | - | Supported since version 4.1.0. CA certificate file in the format `FILE:`. Required when `ssl_mode` is `verify-ca`. Upload the file first using [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md). | +| `slot_name` | No | `doris_cdc_` | Logical replication slot name. The name can contain only lowercase letters, digits, and underscores, cannot start with a digit, and is limited to 63 characters. A custom slot must be created in advance and is not deleted by Doris. | +| `publication_name` | No | `doris_pub_` | Publication name. The naming rules are the same as for `slot_name`. A custom publication must be created in advance, include all synced tables, and is not deleted by Doris. | +| `ssl_mode` | No | `disable` | SSL mode. Valid values are `disable`, `require`, and `verify-ca`. | +| `ssl_rootcert` | Conditionally required | - | CA certificate file in the format `FILE:`. Required when `ssl_mode` is `verify-ca`. Upload the file first using [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md). | If both `include_tables` and `exclude_tables` are set, `include_tables` takes precedence. The following `FROM POSTGRES` parameter fragment syncs `orders` and `customers`, writes `orders` to `ods_orders`, excludes the non-key column `internal_note`, and uses pre-created logical replication resources with CA verification: diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md index 7525b2f24c5843..b1075456f088d4 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md @@ -17,13 +17,17 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + SQL Mapping Sync is implemented through Job + [CDC Stream TVF](../../../../sql-manual/sql-functions/table-valued-functions/cdc-stream.md). The target is an existing Doris table (`INSERT INTO tbl SELECT * FROM cdc_stream(...)`). Doris SQL provides the expressiveness to support column mapping, filtering, and data transformation. This approach is suitable for real-time sync scenarios where data needs to be processed before writing. -> SQL Mapping Sync is supported since version 4.1.0. - By integrating the read capability of [Flink CDC](https://github.com/apache/flink-cdc), Doris reads change logs (WAL) from PostgreSQL to perform full + incremental sync from the source table to the target table. If you want Doris to automatically create downstream tables and sync a group of tables on a per-database basis, see [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md). ### Use Cases @@ -105,7 +109,7 @@ Data source parameters are configured through the `cdc_stream(...)` TVF. They de | `schema` | Yes | - | Schema name. | | `table` | Yes | - | Name of the table to sync. Each SQL Mapping job supports one source table. | | `offset` | Yes | - | Startup offset. `initial`: full and incremental sync; `snapshot`: full sync only; `latest`: sync only changes after the job starts. You can also specify an exact JSON offset, such as `{"lsn":"12345678"}`. PostgreSQL does not support `earliest`. | -| `snapshot_split_size` | No | `40960` (`8096` before 4.1.4) | Split size in rows. During full sync, the table is divided into multiple splits. Must be a positive integer. | +| `snapshot_split_size` | No | `40960` | Split size in rows. During full sync, the table is divided into multiple splits. Must be a positive integer. | | `snapshot_parallelism` | No | `1` | Parallelism of the full-sync phase, that is, the maximum number of splits scheduled by a Task at one time. Must be a positive integer. | | `skip_snapshot_backfill` | No | `false` | Whether to skip WAL backfill during the snapshot. When set to `true`, at-least-once semantics are used. | | `slot_name` | No | `doris_cdc_` | Logical replication slot name. The name can contain only lowercase letters, digits, and underscores, cannot start with a digit, and is limited to 63 characters. A custom slot must be created in advance and is not deleted by Doris. | diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md index 94f56189f2d6e4..fbee49706bbbaa 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +S3 notification ingestion is experimental and supported starting with Doris 4.1.6. + +::: + Notification ingestion uses `NOTIFICATION` to consume S3 object-created events from SQS and continuously ingest new files into Doris. It suits UUID filenames and files arriving in any order, without relying on filename ordering. Only **AWS S3 notifications sent directly to an SQS Standard Queue in the same Region** are supported. Each job needs a dedicated queue. Historical objects are not scanned or backfilled. End-to-end delivery is at-least-once: an object can be ingested more than once, and file ingestion order is not guaranteed. diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md index 77f33f1f6f57e0..4f330a4724c132 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +S3 one-time ingestion is experimental and supported starting with Doris 4.1.6. + +::: + One-time ingestion uses `ONE_TIME` to ingest matching S3 files into an existing Doris table in batches. The job enters `FINISHED` after the final batch commits. It suits historical initialization, a one-off offline load, or a backfill under a separate prefix. Prepare all files before creation and keep the source path stable during ingestion. The completed job does not wait for future files. For continuous ingestion, choose [ordered ingestion](./continuous-load-s3-ordered.md) or [notification ingestion](./continuous-load-s3-notification.md). diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md index 1c47200f75918a..78132515f8a4bd 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + Ordered ingestion uses `ORDERED_LIST` to continuously discover new S3 objects in **lexicographical key order**, including the path and filename. It suits logs and ETL files named with increasing timestamps or sequence numbers. This order describes file discovery and does not guarantee row order in the target table. This is the default S3 ingestion mode and requires no queue. For files arriving in any order, use [notification ingestion](./continuous-load-s3-notification.md). For a prepared static file set, use [one-time ingestion](./continuous-load-s3-one-time.md). diff --git a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3.md b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3.md index 04e8de662c1f8f..936d05dce30d54 100644 --- a/docs/data-operate/import/import-way/streaming-job/continuous-load-s3.md +++ b/docs/data-operate/import/import-way/streaming-job/continuous-load-s3.md @@ -12,7 +12,13 @@ } --- -Doris uses **Streaming Job + S3 TVF** to ingest S3 files into an existing Doris table in batches. Choose ordered, notification, or one-time ingestion based on how files arrive and whether the job should continue running. +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + +Doris uses **Streaming Job + S3 TVF** to ingest S3 files into an existing Doris table in batches. Choose ordered, notification, or one-time ingestion based on how files arrive and whether the job should continue running. Notification and one-time ingestion are supported starting with Doris 4.1.6. ## Choose an ingestion mode diff --git a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md index 38b69074742461..d003c0256a473f 100644 --- a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md +++ b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md @@ -6,6 +6,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + The table below shows how each upstream MySQL column type maps to a Doris column type. There is a single mapping, shared by both sync methods — the only difference is who creates the target table: @@ -17,8 +23,6 @@ In both cases, the values are written following the same mapping. For types that ## JDBC URL Parameter Normalization -> Supported since version 4.1.4. - When you create a Streaming Job with a MySQL data source (or use the `cdc_stream()` table function), Doris automatically appends the following parameters to `jdbc_url` unless you set them explicitly: | Parameter | Injected Value | Effect | @@ -32,7 +36,7 @@ If any of these parameters is set explicitly in `jdbc_url`, Doris keeps your val :::caution Upgrade impact -After upgrading from 4.1.3 or earlier, `TINYINT(1)` columns in existing MySQL CDC jobs are reported as `1` / `0` instead of `true` / `false`, and `YEAR` columns as integer years instead of dates. To keep the previous behavior, set `tinyInt1isBit=true` and `yearIsDateType=true` explicitly in `jdbc_url`. +To read `TINYINT(1)` as `BOOLEAN` or `YEAR` as a date, explicitly set `tinyInt1isBit=true` or `yearIsDateType=true`, respectively, in `jdbc_url`. ::: @@ -40,13 +44,13 @@ After upgrading from 4.1.3 or earlier, `TINYINT(1)` columns in existing MySQL CD | MySQL Type | Doris Type | Notes | | --- | --- | --- | -| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | **Since version 4.1.4** this maps to `TINYINT` (values `1` / `0`); before 4.1.4 it mapped to `BOOLEAN` (`true` / `false`). See [JDBC URL Parameter Normalization](#jdbc-url-parameter-normalization) for the reason | +| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | Maps to `TINYINT` (values `1` / `0`). See [JDBC URL Parameter Normalization](#jdbc-url-parameter-normalization) for the reason | | `TINYINT` | `TINYINT` | `UNSIGNED` → `SMALLINT` | | `SMALLINT` | `SMALLINT` | `UNSIGNED` → `INT` | | `MEDIUMINT` | `INT` | `UNSIGNED` → `INT` | | `INT` | `INT` | `UNSIGNED` → `BIGINT` | | `BIGINT` | `BIGINT` | `UNSIGNED` → `LARGEINT` | -| `YEAR` | `SMALLINT` | **Since version 4.1.4** the value is read as an integer year rather than as a date; the MySQL zero year (`0000`) is kept as `0` | +| `YEAR` | `SMALLINT` | The value is read as an integer year rather than as a date; the MySQL zero year (`0000`) is kept as `0` | | `FLOAT` | `FLOAT` | | | `DOUBLE` | `DOUBLE` | | | `DECIMAL(p,s)` | `DECIMAL(p,s)` | Very high precision falls back to `STRING` | diff --git a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md index 16ea4b15cf2f1e..6d449674101f25 100644 --- a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md +++ b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md @@ -7,16 +7,16 @@ } --- - - -OceanBase CDC continuous load supports only MySQL compatibility mode and uses the MySQL type conversion path. The following table shows how OceanBase column types map to Doris column types during automatic table creation. - :::caution Experimental feature -OceanBase CDC continuous load is available as an experimental feature starting from Doris 4.1.4. +Doris continuous load is experimental and supported starting with Doris 4.1.5. ::: + + +OceanBase CDC continuous load supports only MySQL compatibility mode and uses the MySQL type conversion path. The following table shows how OceanBase column types map to Doris column types during automatic table creation. + ## JDBC URL Parameter Normalization When you create an OceanBase Streaming Job, Doris automatically appends the following parameters to `jdbc_url` unless you set them explicitly: diff --git a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md index bf89b8b68fdf8b..cf7ececf50b509 100644 --- a/docs/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md +++ b/docs/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md @@ -6,6 +6,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + The table below shows how each upstream PostgreSQL column type maps to a Doris column type. There is a single mapping, shared by both sync methods — the only difference is who creates the target table: diff --git a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md index 715d55dbe79e36..6f9e3939aa7f88 100644 --- a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md +++ b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md index 97634392e379ef..3b1c6c3d178b32 100644 --- a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md +++ b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -142,18 +148,6 @@ Privilege description: -### Doris 4.0 - -A Publication named `dbz_publication` must be created in advance with `FOR ALL TABLES`: - -```sql -CREATE PUBLICATION dbz_publication FOR ALL TABLES; -``` - -If the sync user has superuser privileges, such as the `rds_superuser` role, Doris can automatically create `dbz_publication` when the task starts, and you do not need to execute this statement manually. Doris 4.0 does not support custom Publication names or Publications that contain only a subset of tables. - -### Doris 4.1.0 and later - By default, you do not need to create a Publication manually. When `publication_name` is not set, Doris uses the sync account to create a Publication named `doris_pub_` and cleans it up when the job is deleted. The sync account therefore needs permission to create Publications. If `publication_name` is explicitly set when creating the Streaming Job, create a Publication with the same name in advance. A custom Publication must include all source tables synchronized by the job and must be maintained and cleaned up by the user. For example: @@ -179,11 +173,11 @@ No. Doris streaming ingestion requires Aurora PostgreSQL 14 or later. **Q3: Can I use a Publication with a custom name?** -Doris 4.0 does not support custom names and only supports `dbz_publication FOR ALL TABLES`. Since version 4.1.0, you can specify a custom name through `publication_name`, but you must create the Publication before creating the job and ensure that it includes all synchronized tables. +You can specify a custom name through `publication_name`, but you must create the Publication before creating the job and ensure that it includes all synchronized tables. **Q4: Must I use a superuser account for synchronization?** -No. In Doris 4.0, a regular sync user requires `dbz_publication` to be created manually, while Doris can create it automatically when the user has superuser privileges. Since version 4.1.0, when `publication_name` is omitted, Doris automatically creates and manages the default Publication. A Publication must be created in advance only when a custom Publication is used. +No. When `publication_name` is omitted, Doris automatically creates and manages the default Publication. A Publication must be created in advance only when a custom Publication is used. ## Troubleshooting @@ -195,6 +189,5 @@ No. In Doris 4.0, a regular sync user requires `dbz_publication` to be created m | `SHOW rds.logical_replication;` still returns `off` | The new parameter group is not bound to the cluster, or the writer instance has not been restarted | Verify that **DB cluster parameter group** has been switched to the new parameter group, and restart the writer instance | | User creation or privilege grant fails | The current login account has insufficient privileges | Sign in with an account that has `rds_superuser` or equivalent privileges, then perform the operation | | `GRANT rds_replication` returns an error | The current Aurora version is too low, or the cluster is not Aurora PostgreSQL | Upgrade to Aurora PostgreSQL 14 or later | -| Doris 4.0 reports that `dbz_publication` does not exist | `dbz_publication` has not been created, and the sync account lacks superuser privileges | Execute `CREATE PUBLICATION dbz_publication FOR ALL TABLES;`, or use an account with the required privileges | -| Doris 4.1.0 or later fails to create the default Publication | The sync account lacks permission to create Publications | Grant the required privileges to the sync account, or have an administrator create a custom Publication in advance and configure `publication_name` | -| Doris 4.1.0 or later reports that a custom Publication does not exist | `publication_name` is configured, but a Publication with the same name has not been created | Create the Publication as described in [Step 5](#step-5-configure-the-publication), or remove the `publication_name` configuration | +| The default Publication cannot be created | The sync account lacks permission to create Publications | Grant the required privileges to the sync account, or have an administrator create a custom Publication in advance and configure `publication_name` | +| A custom Publication does not exist | `publication_name` is configured, but a Publication with the same name has not been created | Create the Publication as described in [Step 5](#step-5-configure-the-publication), or remove the `publication_name` configuration | diff --git a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md index 3cefe58c4d4a5c..e628df91b71032 100644 --- a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md +++ b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md index b5539bc814d34a..6718569cb99b94 100644 --- a/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md +++ b/docs/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md @@ -17,6 +17,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -159,18 +165,6 @@ Privilege overview: -### Doris 4.0 - -A Publication named `dbz_publication` must be created in advance with `FOR ALL TABLES`: - -```sql -CREATE PUBLICATION dbz_publication FOR ALL TABLES; -``` - -If the sync user has superuser privileges, such as the `rds_superuser` role, Doris can automatically create `dbz_publication` when the task starts, and you do not need to execute this statement manually. Doris 4.0 does not support custom Publication names or Publications that contain only a subset of tables. - -### Doris 4.1.0 and later - By default, you do not need to create a Publication manually. When `publication_name` is not set, Doris uses the sync account to create a Publication named `doris_pub_` and cleans it up when the job is deleted. The sync account therefore needs permission to create Publications. If `publication_name` is explicitly set when creating the Streaming Job, create a Publication with the same name in advance. A custom Publication must include all source tables synchronized by the job and must be maintained and cleaned up by the user. For example: @@ -194,7 +188,7 @@ No. `on` means logical replication is already in effect, and you can skip direct **Q2: Can I use a Publication with a different name?** -Doris 4.0 does not support custom names and only supports `dbz_publication FOR ALL TABLES`. Since version 4.1.0, you can specify a custom name through `publication_name`, but you must create the Publication before creating the job and ensure that it includes all synchronized tables. +You can specify a custom name through `publication_name`, but you must create the Publication before creating the job and ensure that it includes all synchronized tables. **Q3: Is it required to use the `rds_replication` role?** @@ -202,7 +196,7 @@ Yes. For security reasons, RDS PostgreSQL does not allow using the standard Post **Q4: Is it mandatory to manually create the Publication?** -In Doris 4.0, a regular sync user must create `dbz_publication` manually, while Doris can create it automatically when the user has superuser privileges. Since version 4.1.0, when `publication_name` is omitted, Doris automatically creates and manages the default Publication. Only a custom Publication must be created in advance. For details, see [Step 5](#step-5-configure-the-publication). +When `publication_name` is omitted, Doris automatically creates and manages the default Publication. Only a custom Publication must be created in advance. For details, see [Step 5](#step-5-configure-the-publication). --- @@ -215,7 +209,6 @@ In Doris 4.0, a regular sync user must create `dbz_publication` manually, while | `SHOW rds.logical_replication;` still returns `off` | The parameter group is not bound to the instance, or the instance has not been restarted | Check whether the instance uses the new parameter group, and restart it | | Permission error when creating the user | The current account does not have sufficient privileges | Use an administrator account with the privilege to create users | | `GRANT rds_replication` returns an error | The current account does not have permission to grant this role | Use an `rds_superuser` account or another account with the grant privilege | -| Doris 4.0 reports that `dbz_publication` does not exist | `dbz_publication` has not been created, and the sync account lacks superuser privileges | Execute `CREATE PUBLICATION dbz_publication FOR ALL TABLES;`, or use an account with the required privileges | -| Doris 4.1.0 or later fails to create the default Publication | The sync account lacks permission to create Publications | Grant the required privileges to the sync account, or have an administrator create a custom Publication in advance and configure `publication_name` | -| Doris 4.1.0 or later reports that a custom Publication does not exist | `publication_name` is configured, but a Publication with the same name has not been created | Create the Publication as described in [Step 5](#step-5-configure-the-publication), or remove `publication_name` to use the default Publication managed by Doris | +| The default Publication cannot be created | The sync account lacks permission to create Publications | Grant the required privileges to the sync account, or have an administrator create a custom Publication in advance and configure `publication_name` | +| A custom Publication does not exist | `publication_name` is configured, but a Publication with the same name has not been created | Create the Publication as described in [Step 5](#step-5-configure-the-publication), or remove `publication_name` to use the default Publication managed by Doris | | Parameter changes do not take effect | **Apply immediately** was not selected, or the instance was not restarted | Re-execute [Step 3](#step-3-apply-the-parameter-group-and-restart) | diff --git a/docs/data-operate/import/import-way/streaming-job/schema-change-mysql.md b/docs/data-operate/import/import-way/streaming-job/schema-change-mysql.md index 31ca1d6de1b508..9575d2f49d7fc9 100644 --- a/docs/data-operate/import/import-way/streaming-job/schema-change-mysql.md +++ b/docs/data-operate/import/import-way/streaming-job/schema-change-mysql.md @@ -16,16 +16,16 @@ } --- - - +:::caution Experimental feature -MySQL Schema Change Sync automatically applies upstream MySQL column changes to Doris during continuous load. This capability applies only to [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md). It is not supported by [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md). +Doris continuous load is experimental and supported starting with Doris 4.1.5. -:::tip +::: -MySQL Schema Change Sync is supported from Doris 4.1.4. + + -::: +MySQL Schema Change Sync automatically applies upstream MySQL column changes to Doris during continuous load. This capability applies only to [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md). It is not supported by [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md). ## Supported Schema Changes diff --git a/docs/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md b/docs/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md index 958b3be3531cc0..3bd25cba8dd081 100644 --- a/docs/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md +++ b/docs/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md @@ -15,16 +15,18 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + OceanBase Schema Change Sync automatically applies upstream column changes to Doris target tables during continuous load. This capability applies only to [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md). -:::caution Experimental feature - -OceanBase Schema Change Sync is available as an experimental feature starting from Doris 4.1.4 and supports only MySQL compatibility mode. - -::: +OceanBase Schema Change Sync supports only MySQL compatibility mode. OceanBase uses the MySQL CDC schema change processing path, so its supported changes and behavior are the same as for MySQL. diff --git a/docs/data-operate/import/import-way/streaming-job/schema-change-postgresql.md b/docs/data-operate/import/import-way/streaming-job/schema-change-postgresql.md index bb5d9031dc2d55..abab47e15d4239 100644 --- a/docs/data-operate/import/import-way/streaming-job/schema-change-postgresql.md +++ b/docs/data-operate/import/import-way/streaming-job/schema-change-postgresql.md @@ -16,16 +16,16 @@ } --- - - +:::caution Experimental feature -PostgreSQL Schema Change Sync automatically applies upstream PostgreSQL column changes to Doris during continuous load. This capability applies only to [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md). It is not supported by [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md). +Doris continuous load is experimental and supported starting with Doris 4.1.5. -:::tip +::: -PostgreSQL Schema Change Sync is supported from Doris 4.1.0. + + -::: +PostgreSQL Schema Change Sync automatically applies upstream PostgreSQL column changes to Doris during continuous load. This capability applies only to [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md). It is not supported by [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md). ## Supported Schema Changes @@ -43,14 +43,14 @@ PostgreSQL Schema Change Sync is supported from Doris 4.1.0. - Column renames, column type changes, DEFAULT changes, and `NULL` / `NOT NULL` constraint changes are not synchronized automatically. Pause the continuous load job, change the Doris target table manually, verify that both schemas are compatible, and then resume the job. - Auto Table Creation Sync enables Schema Change Sync by default. Doris currently does not provide a SQL property to disable it. The `cdc_stream()` table function (SQL Mapping Sync) disables Schema Change Sync internally. -:::caution Behavior change (4.1.4) +:::caution Schema change behavior -Since version 4.1.4, PostgreSQL schema change detection is driven by Relation events: +PostgreSQL schema change detection is driven by Relation events: - Only `ADD COLUMN` and `DROP COLUMN` are recognized. - A change that adds and drops columns at the same time (possibly a `RENAME`) is skipped, and the target table is left unchanged. - Column type changes are skipped. -- An added column **no longer carries the `DEFAULT` value of the upstream column**. +- An added column **does not carry the `DEFAULT` value of the upstream column**. - This capability applies only to the Auto Table Creation Sync (at-least-once) path. The SQL Mapping (TVF / exactly-once) path does not support it. ::: diff --git a/i18n/ja/docusaurus-plugin-content-docs/current.json b/i18n/ja/docusaurus-plugin-content-docs/current.json index 509249159de4e6..035900a38e0f7b 100644 --- a/i18n/ja/docusaurus-plugin-content-docs/current.json +++ b/i18n/ja/docusaurus-plugin-content-docs/current.json @@ -131,21 +131,21 @@ "message": "継続的なロード", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL（実験的機能）", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "前提条件", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL（実験的機能）", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase（実験的機能）", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.File Formats": { "message": "ファイル形式", diff --git a/i18n/ja/docusaurus-plugin-content-docs/version-4.x.json b/i18n/ja/docusaurus-plugin-content-docs/version-4.x.json index aee31e9a752da1..32285e49a7973f 100644 --- a/i18n/ja/docusaurus-plugin-content-docs/version-4.x.json +++ b/i18n/ja/docusaurus-plugin-content-docs/version-4.x.json @@ -127,21 +127,21 @@ "message": "継続的なロード", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL（実験的機能）", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "前提条件", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL（実験的機能）", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase（実験的機能）", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.File Formats": { "message": "ファイル形式", diff --git a/i18n/ko/docusaurus-plugin-content-docs/current.json b/i18n/ko/docusaurus-plugin-content-docs/current.json index 7af91ea644d3e4..d86b685f0acc1f 100644 --- a/i18n/ko/docusaurus-plugin-content-docs/current.json +++ b/i18n/ko/docusaurus-plugin-content-docs/current.json @@ -135,21 +135,21 @@ "message": "지속적 데이터 적재", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL (실험적)", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "사전 요구 사항", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL (실험적)", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase (실험적)", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.S3": { "message": "S3", diff --git a/i18n/ko/docusaurus-plugin-content-docs/version-4.x.json b/i18n/ko/docusaurus-plugin-content-docs/version-4.x.json index b7e13d167874a0..1a186fb7ca9219 100644 --- a/i18n/ko/docusaurus-plugin-content-docs/version-4.x.json +++ b/i18n/ko/docusaurus-plugin-content-docs/version-4.x.json @@ -131,21 +131,21 @@ "message": "지속적 데이터 적재", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL (실험적)", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "사전 요구 사항", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL (실험적)", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase (실험적)", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.S3": { "message": "S3", diff --git a/i18n/ru/docusaurus-plugin-content-docs/current.json b/i18n/ru/docusaurus-plugin-content-docs/current.json index 5446b6474e7894..bfc794e94392a9 100644 --- a/i18n/ru/docusaurus-plugin-content-docs/current.json +++ b/i18n/ru/docusaurus-plugin-content-docs/current.json @@ -135,21 +135,21 @@ "message": "Непрерывная загрузка", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL (экспериментально)", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "Предварительные требования", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL (экспериментально)", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase (экспериментально)", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.S3": { "message": "S3", diff --git a/i18n/ru/docusaurus-plugin-content-docs/version-4.x.json b/i18n/ru/docusaurus-plugin-content-docs/version-4.x.json index 5de942e203bdfc..d143f8b627ed76 100644 --- a/i18n/ru/docusaurus-plugin-content-docs/version-4.x.json +++ b/i18n/ru/docusaurus-plugin-content-docs/version-4.x.json @@ -131,21 +131,21 @@ "message": "Непрерывная загрузка", "description": "The label for category Continuous Load in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL (экспериментально)", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Prerequisites": { "message": "Предварительные требования", "description": "The label for category Prerequisites in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL (экспериментально)", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase (экспериментально)", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.S3": { "message": "S3", diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current.json b/i18n/zh-CN/docusaurus-plugin-content-docs/current.json index bcf2196b8e3f42..57d1e0d93c9be8 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current.json +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current.json @@ -735,17 +735,17 @@ "message": "更多", "description": "The label for category More in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL（实验性）", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Optimization Technology Principles": { "message": "优化技术原理", "description": "The label for category Optimization Technology Principles in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase（实验性）", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.Paimon Catalog": { "message": "Paimon Catalog", @@ -755,9 +755,9 @@ "message": "性能优化", "description": "The label for category Performance in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL（实验性）", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, "sidebar.docs.category.Query Performance": { "message": "查询性能", diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md index 036e2dcc77fc65..41672149fc49f0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -145,15 +151,15 @@ MySQL 源端（`FROM MYSQL`）支持以下参数。连接信息、数据库名 | `database` | 是 | - | MySQL 数据库名。 | | `include_tables` | 否 | - | 需要同步的表名，多个表用逗号分隔；不设置时同步数据库中的所有表。 | | `exclude_tables` | 否 | - | 不同步的表名，多个表用逗号分隔。仅当未设置 `include_tables` 时生效。 | -| `table..target_table` | 否 | 源表名 | 自 4.1.0 版本起支持。为指定源表设置 Doris 目标表名，`` 使用源表名。 | -| `table..exclude_columns` | 否 | - | 自 4.1.0 版本起支持。指定源表中不同步的列，多个列用逗号分隔。列必须存在，且不能排除主键列。 | -| `offset` | 否 | `latest` | 启动位点。`initial`：全量 + 增量同步；`earliest`：从当前可用的最早 Binlog 位点开始；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"file":"binlog.000001","pos":"154"}` 或 `{"gtids":""}`。自 4.1.0 版本起支持 `snapshot`，表示仅全量同步。 | +| `table..target_table` | 否 | 源表名 | 为指定源表设置 Doris 目标表名，`` 使用源表名。 | +| `table..exclude_columns` | 否 | - | 指定源表中不同步的列，多个列用逗号分隔。列必须存在，且不能排除主键列。 | +| `offset` | 否 | `latest` | 启动位点。`initial`：全量 + 增量同步；`earliest`：从当前可用的最早 Binlog 位点开始；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"file":"binlog.000001","pos":"154"}` 或 `{"gtids":""}`。支持 `snapshot`，表示仅全量同步。 | | `schema_change_behavior` | 否 | `evolve` | 字符串，不区分大小写。`evolve` 自动同步支持的列变更，`ignore` 关闭结构同步。可暂停后通过 `ALTER JOB` 修改；支持范围及暂停处理见 [Schema Change 同步](./schema-change-mysql.md)。 | -| `snapshot_split_size` | 否 | `40960`（4.1.4 之前为 `8096`） | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | +| `snapshot_split_size` | 否 | `40960` | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | | `snapshot_parallelism` | 否 | `1` | 全量阶段的并行度，即单次 Task 最多调度的 split 数量。必须为正整数。 | -| `server_id` | 否 | 自动生成 | 自 4.1.0 版本起支持。MySQL CDC reader 的 server ID。支持单值（如 `5400`）或闭区间（如 `5400-5408`）；区间宽度必须大于等于 `snapshot_parallelism`。 | -| `ssl_mode` | 否 | `disable` | 自 4.1.0 版本起支持。SSL 模式，可选值为 `disable`、`require`、`verify-ca`。 | -| `ssl_rootcert` | 条件必填 | - | 自 4.1.0 版本起支持。CA 证书文件，格式为 `FILE:`。`ssl_mode` 为 `verify-ca` 时必填；文件需先通过 [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md) 上传。 | +| `server_id` | 否 | 自动生成 | MySQL CDC reader 的 server ID。支持单值（如 `5400`）或闭区间（如 `5400-5408`）；区间宽度必须大于等于 `snapshot_parallelism`。 | +| `ssl_mode` | 否 | `disable` | SSL 模式，可选值为 `disable`、`require`、`verify-ca`。 | +| `ssl_rootcert` | 条件必填 | - | CA 证书文件，格式为 `FILE:`。`ssl_mode` 为 `verify-ca` 时必填；文件需先通过 [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md) 上传。 | 如果同时设置 `include_tables` 和 `exclude_tables`，以 `include_tables` 为准。以下 `FROM MYSQL` 参数片段同步 `orders` 和 `customers`，将 `orders` 写入 `ods_orders` 并排除非主键列 `internal_note`，同时启用 CA 校验： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md index 29af49f4cbf65f..ab620b1a3ee6d6 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md @@ -17,13 +17,17 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + SQL 映射同步通过 Job + [CDC Stream TVF](../../../../sql-manual/sql-functions/table-valued-functions/cdc-stream.md) 实现，目标是一张**已存在**的 Doris 表（`INSERT INTO tbl SELECT * FROM cdc_stream(...)`）。借助 Doris SQL 的表达能力，可以在同步链路中进行列映射、过滤和数据转换，适用于对数据需要做加工的实时同步场景。 -> SQL 映射同步自 4.1.0 版本起支持。 - 通过集成 [Flink CDC](https://github.com/apache/flink-cdc) 的读取能力，Doris 从 MySQL 读取变更日志（Binlog），完成源表到目标表的**全量 + 增量**同步。若希望 Doris 自动创建下游表、按库为单位同步一组表，请参考 [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md)。 ### 适用场景 @@ -36,7 +40,7 @@ SQL 映射同步通过 Job + [CDC Stream TVF](../../../../sql-manual/sql-functio | 检查项 | 说明 | | ------ | ---- | -| Doris 版本 | 4.1.0 及以上 | +| Doris 版本 | 4.1.5 及以上 | | 表类型 | 上游源表必须有主键；Doris 目标表仅支持主键模型表（Unique Key） | | 用户权限 | 需要 `Load` 权限 | | MySQL 配置 | 需要在 MySQL 端开启 Binlog，参考[配置指南](./continuous-load-overview.md#支持的数据源与同步模式) | @@ -95,7 +99,7 @@ CDC Stream TVF 支持以下 MySQL 数据源参数。 | `database` | 是 | - | MySQL 数据库名。 | | `table` | 是 | - | 需要同步的表名。SQL Mapping 每个作业只支持一张源表。 | | `offset` | 是 | - | 启动位点。`initial`：全量 + 增量同步；`snapshot`：仅全量同步；`earliest`：从当前可用的最早 Binlog 位点开始；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"file":"binlog.000001","pos":"154"}` 或 `{"gtids":""}`。 | -| `snapshot_split_size` | 否 | `40960`（4.1.4 之前为 `8096`） | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | +| `snapshot_split_size` | 否 | `40960` | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | | `snapshot_parallelism` | 否 | `1` | 全量阶段的并行度，即单次 Task 最多调度的 split 数量。必须为正整数。 | | `skip_snapshot_backfill` | 否 | `false` | 是否跳过快照期间的 Binlog 回填。设置为 `true` 时采用 at-least-once 语义。 | | `server_id` | 否 | 自动生成 | MySQL CDC reader 的 server ID。支持单值（如 `5400`）或闭区间（如 `5400-5408`）；区间宽度必须大于等于 `snapshot_parallelism`。 | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md index f22d16727566cf..ef24d76c844088 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md @@ -16,16 +16,18 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + OceanBase CDC 自动建表同步通过 `FROM OCEANBASE (...) TO DATABASE (...)` 实现。Doris 从 OBBinlog 服务读取 OceanBase 的全量和增量数据，并根据上游表结构自动创建 Doris 主键表。可以通过 `include_tables` 同步一张、多张或全部表。 -:::caution 实验性功能 - -OceanBase CDC 持续导入自 Doris 4.1.4 起作为实验性功能提供。目前仅支持 OceanBase 的 MySQL 兼容模式和自动建表同步，不支持 Oracle 兼容模式与 SQL 映射同步。 - -::: +目前仅支持 OceanBase 的 MySQL 兼容模式和自动建表同步，不支持 Oracle 兼容模式与 SQL 映射同步。 ### 适用场景 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-overview.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-overview.md index eafb009b7e3efb..10bba96c9c2430 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-overview.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-overview.md @@ -20,15 +20,17 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + Doris 支持通过 **Streaming Job** 的方式，从多种数据源持续导入数据到 Doris 表中。提交 Job 后，Doris 会持续运行导入作业，实时读取数据源中的数据并写入到 Doris 表中。 -:::tip -该功能自 4.1.0 版本起支持。 -::: - 本文将帮助你解决以下问题： - 持续导入支持哪些数据源和同步模式？ @@ -39,13 +41,13 @@ Doris 支持通过 **Streaming Job** 的方式，从多种数据源持续导入 ## 支持的数据源与同步模式 -持续导入支持以下数据源和同步模式： +持续导入支持以下数据源和同步模式。S3 通知导入和一次性导入自 Doris 4.1.6 版本起支持： | 数据源 | 支持版本 | SQL 映射同步 | 自动建表同步 | 配置指南 | | :--------- | :---------------- | :-------------------------------------------------------------------- | :-------------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | | MySQL | 5.6、5.7、8.0.x | [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md) | [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md) | [Amazon RDS MySQL](./prerequisites/amazon-rds-mysql.md) · [Amazon Aurora MySQL](./prerequisites/amazon-aurora-mysql.md) | | PostgreSQL | 14、15、16、17 | [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md) | [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md) | [Amazon RDS PostgreSQL](./prerequisites/amazon-rds-postgresql.md) · [Amazon Aurora PostgreSQL](./prerequisites/amazon-aurora-postgresql.md) | -| OceanBase | MySQL 兼容模式 | - | [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md)（自 4.1.4 版本起） | - | +| OceanBase | MySQL 兼容模式 | - | [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md) | - | | S3 | - | [S3 导入概览](./continuous-load-s3.md) | - | - | 上游列类型如何映射为 Doris 类型，见 [MySQL](./data-type-mapping-mysql.md)、[PostgreSQL](./data-type-mapping-postgresql.md)和 [OceanBase](./data-type-mapping-oceanbase.md) 数据类型映射。 @@ -146,8 +148,8 @@ select * from jobs("type"="insert") where ExecuteType = "STREAMING"; | LoadStatistic | Job 的统计信息 | | ErrorMsg | Job 执行的错误信息 | | JobRuntimeMsg | Job 运行时的一些提示信息 | -| LagBytes | 数据源端日志（MySQL / OceanBase Binlog、PostgreSQL WAL）的积压字节数，`-1` 表示当前不可用（例如 S3 数据源或全量快照阶段）。**自 4.1.4 版本起**，该列由原来的 `Lag`（单位：秒）改名为 `LagBytes`（单位：字节） | -| LastSourceEventTimestamp | 已提交 Offset 中记录的数据源端最新事件时间戳（Unix 秒），为空表示不可用。**自 4.1.4 版本起新增** | +| LagBytes | 数据源端日志（MySQL / OceanBase Binlog、PostgreSQL WAL）的积压字节数，`-1` 表示当前不可用（例如 S3 数据源或全量快照阶段） | +| LastSourceEventTimestamp | 已提交 Offset 中记录的数据源端最新事件时间戳（Unix 秒），为空表示不可用 | | LastTaskSuccessTime | 最近一次 Task 成功完成的时间 | ### 查看 Task 状态 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md index 8747a22daee73f..80cab6e69f71ce 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md @@ -19,6 +19,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -96,16 +102,16 @@ PostgreSQL 源端参数用于配置 JDBC 连接、同步范围以及全量切片 | `schema` | 是 | - | Schema 名称。 | | `include_tables` | 否 | - | 需要同步的表名，多个表用逗号分隔；不设置时同步 Schema 中的所有表。 | | `exclude_tables` | 否 | - | 不同步的表名，多个表用逗号分隔。仅当未设置 `include_tables` 时生效。 | -| `table..target_table` | 否 | 源表名 | 自 4.1.0 版本起支持。为指定源表设置 Doris 目标表名，`` 使用源表名。 | -| `table..exclude_columns` | 否 | - | 自 4.1.0 版本起支持。指定源表中不同步的列，多个列用逗号分隔。列必须存在，且不能排除主键列。 | -| `offset` | 否 | `latest` | 启动位点。`initial`：全量 + 增量同步；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"lsn":"12345678"}`。自 4.1.0 版本起支持 `snapshot`，表示仅全量同步。PostgreSQL 不支持 `earliest`。 | +| `table..target_table` | 否 | 源表名 | 为指定源表设置 Doris 目标表名，`` 使用源表名。 | +| `table..exclude_columns` | 否 | - | 指定源表中不同步的列，多个列用逗号分隔。列必须存在，且不能排除主键列。 | +| `offset` | 否 | `latest` | 启动位点。`initial`：全量 + 增量同步；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"lsn":"12345678"}`。支持 `snapshot`，表示仅全量同步。PostgreSQL 不支持 `earliest`。 | | `schema_change_behavior` | 否 | `evolve` | 字符串，不区分大小写。`evolve` 自动同步支持的列变更，`ignore` 关闭结构同步。可暂停后通过 `ALTER JOB` 修改；支持范围及暂停处理见 [Schema Change 同步](./schema-change-postgresql.md)。 | -| `snapshot_split_size` | 否 | `40960`（4.1.4 之前为 `8096`） | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | +| `snapshot_split_size` | 否 | `40960` | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | | `snapshot_parallelism` | 否 | `1` | 全量阶段的并行度，即单次 Task 最多调度的 split 数量。必须为正整数。 | -| `slot_name` | 否 | `doris_cdc_` | 自 4.1.0 版本起支持。逻辑复制槽名称。名称只能包含小写字母、数字和下划线，不能以数字开头，最长 63 个字符。自定义复制槽必须预先创建，且不会由 Doris 删除。 | -| `publication_name` | 否 | `doris_pub_` | 自 4.1.0 版本起支持。发布名称，命名规则与 `slot_name` 相同。自定义发布必须预先创建并包含所有同步表，且不会由 Doris 删除。 | -| `ssl_mode` | 否 | `disable` | 自 4.1.0 版本起支持。SSL 模式，可选值为 `disable`、`require`、`verify-ca`。 | -| `ssl_rootcert` | 条件必填 | - | 自 4.1.0 版本起支持。CA 证书文件，格式为 `FILE:`。`ssl_mode` 为 `verify-ca` 时必填；文件需先通过 [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md) 上传。 | +| `slot_name` | 否 | `doris_cdc_` | 逻辑复制槽名称。名称只能包含小写字母、数字和下划线，不能以数字开头，最长 63 个字符。自定义复制槽必须预先创建，且不会由 Doris 删除。 | +| `publication_name` | 否 | `doris_pub_` | 发布名称，命名规则与 `slot_name` 相同。自定义发布必须预先创建并包含所有同步表，且不会由 Doris 删除。 | +| `ssl_mode` | 否 | `disable` | SSL 模式，可选值为 `disable`、`require`、`verify-ca`。 | +| `ssl_rootcert` | 条件必填 | - | CA 证书文件，格式为 `FILE:`。`ssl_mode` 为 `verify-ca` 时必填；文件需先通过 [CREATE FILE](../../../../sql-manual/sql-statements/security/CREATE-FILE.md) 上传。 | 如果同时设置 `include_tables` 和 `exclude_tables`，以 `include_tables` 为准。以下 `FROM POSTGRES` 参数片段同步 `orders` 和 `customers`，将 `orders` 写入 `ods_orders` 并排除非主键列 `internal_note`，同时使用预先创建的逻辑复制资源和 CA 校验： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md index ce0913669b537d..7f78b5b2a780fa 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md @@ -17,13 +17,17 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + SQL 映射同步通过 Job + [CDC Stream TVF](../../../../sql-manual/sql-functions/table-valued-functions/cdc-stream.md) 实现，目标是一张已存在的 Doris 表（`INSERT INTO tbl SELECT * FROM cdc_stream(...)`），借助 Doris SQL 的表达能力支持列映射、过滤和数据转换，适用于对数据需要做加工的实时同步场景。 -> SQL 映射同步自 4.1.0 版本起支持。 - 通过集成 [Flink CDC](https://github.com/apache/flink-cdc) 的读取能力，Doris 从 PostgreSQL 读取变更日志（WAL），实现源表到目标表的全量 + 增量同步。若希望 Doris 自动创建下游表、按库为单位同步一组表，请参考 [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md)。 ### 适用场景 @@ -105,7 +109,7 @@ select * from jobs("type"="insert") where ExecuteType = "STREAMING"; | `schema` | 是 | - | Schema 名称。 | | `table` | 是 | - | 需要同步的表名。SQL Mapping 每个作业只支持一张源表。 | | `offset` | 是 | - | 启动位点。`initial`：全量 + 增量同步；`snapshot`：仅全量同步；`latest`：仅同步作业启动后的增量；也可设置 JSON 精确位点，例如 `{"lsn":"12345678"}`。PostgreSQL 不支持 `earliest`。 | -| `snapshot_split_size` | 否 | `40960`（4.1.4 之前为 `8096`） | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | +| `snapshot_split_size` | 否 | `40960` | split 的大小（行数）。全量同步时，表会被切分成多个 split。必须为正整数。 | | `snapshot_parallelism` | 否 | `1` | 全量阶段的并行度，即单次 Task 最多调度的 split 数量。必须为正整数。 | | `skip_snapshot_backfill` | 否 | `false` | 是否跳过快照期间的 WAL 回填。设置为 `true` 时采用 at-least-once 语义。 | | `slot_name` | 否 | `doris_cdc_` | 逻辑复制槽名称。名称只能包含小写字母、数字和下划线，不能以数字开头，最长 63 个字符。自定义复制槽必须预先创建，且不会由 Doris 删除。 | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md index c2aec48bfa5170..078de88aec2016 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +S3 通知导入为实验性功能，自 Doris 4.1.6 版本起支持。 + +::: + 通知导入通过 `NOTIFICATION` 模式，消费 SQS 中的 S3 对象创建事件，将新文件持续导入 Doris。适用于 UUID 文件名、乱序到达或文件名无法单调递增的场景，无需依赖文件名排序。 目前仅支持 **AWS S3 直接发送到同 Region 的 SQS Standard Queue**。每个作业需要独占队列；只处理队列中的通知，不补导历史对象。端到端采用 at-least-once 语义，可能重复导入同一对象，且不保证文件导入顺序。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md index 1f716824b6b0ac..b055a779656259 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +S3 一次性导入为实验性功能，自 Doris 4.1.6 版本起支持。 + +::: + 一次性导入通过 `ONE_TIME` 模式，将指定 S3 路径中匹配的文件分批导入已有的 Doris 表，最后一批事务提交后作业进入 `FINISHED`。适用于静态历史数据初始化、一次性的离线数据入仓或独立前缀下的回补任务。 创建前应准备好所有文件，导入期间保持源路径稳定。作业不会在完成后持续等待新文件；如需持续接入，使用[有序导入](./continuous-load-s3-ordered.md)或[通知导入](./continuous-load-s3-notification.md)。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md index d2e5a7de03f783..63fb4072844583 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 有序导入通过 `ORDERED_LIST` 模式，按 S3 对象 key（文件名及路径）的**字典序**持续发现新文件，适用于按时间戳或递增序号命名的日志、埋点及 ETL 文件。这里的“有序”指文件发现规则，不表示目标表中的行具有顺序保证。 这是默认的 S3 导入模式，无需配置消息队列。若文件可能乱序到达，使用[通知导入](./continuous-load-s3-notification.md)；若只需导入一批已准备好的文件，使用[一次性导入](./continuous-load-s3-one-time.md)。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3.md index c6a6fef627eee7..de5481bb4fed8a 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/continuous-load-s3.md @@ -12,7 +12,13 @@ } --- -Doris 通过 **Streaming Job + S3 TVF** 将 S3 文件分批导入已有的 Doris 表。根据文件生成方式和是否需要持续运行，选择有序导入、通知导入或一次性导入。 +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + +Doris 通过 **Streaming Job + S3 TVF** 将 S3 文件分批导入已有的 Doris 表。根据文件生成方式和是否需要持续运行，选择有序导入、通知导入或一次性导入。通知导入和一次性导入自 Doris 4.1.6 版本起支持。 ## 选择导入模式 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md index 300fba095ed9da..8a483ad034fb25 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md @@ -6,6 +6,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 下表展示上游 MySQL 的每个列类型如何映射为 Doris 列类型。**只有一套映射**，两种同步方式共用，区别只在于由谁创建目标表： @@ -17,8 +23,6 @@ ## JDBC URL 参数归一化 -> 自 4.1.4 版本开始生效。 - 创建 MySQL 数据源的 Streaming Job（以及使用 `cdc_stream()` 表函数）时，如果用户没有显式指定，Doris 会自动在 `jdbc_url` 上追加以下参数： | 参数 | 注入值 | 影响 | @@ -32,7 +36,7 @@ :::caution 升级影响 -从 4.1.3 及更早版本升级后，已有的 MySQL CDC 作业中 `TINYINT(1)` 列的取值展示会由 `true` / `false` 变为 `1` / `0`，`YEAR` 列会由日期变为整数年份。如果需要保留旧行为，可在 `jdbc_url` 中显式设置 `tinyInt1isBit=true` 和 `yearIsDateType=true`。 +如需将 `TINYINT(1)` 按 `BOOLEAN` 读取，或将 `YEAR` 按日期读取，可在 `jdbc_url` 中分别显式设置 `tinyInt1isBit=true` 和 `yearIsDateType=true`。 ::: @@ -40,13 +44,13 @@ | MySQL 类型 | Doris 类型 | 备注 | | --- | --- | --- | -| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | **自 4.1.4 版本起**映射为 `TINYINT`（取值 `1` / `0`）；4.1.4 之前映射为 `BOOLEAN`（`true` / `false`）。原因见 [JDBC URL 参数归一化](#jdbc-url-参数归一化) | +| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | 映射为 `TINYINT`（取值 `1` / `0`）。原因见 [JDBC URL 参数归一化](#jdbc-url-参数归一化) | | `TINYINT` | `TINYINT` | `UNSIGNED` → `SMALLINT` | | `SMALLINT` | `SMALLINT` | `UNSIGNED` → `INT` | | `MEDIUMINT` | `INT` | `UNSIGNED` → `INT` | | `INT` | `INT` | `UNSIGNED` → `BIGINT` | | `BIGINT` | `BIGINT` | `UNSIGNED` → `LARGEINT` | -| `YEAR` | `SMALLINT` | **自 4.1.4 版本起**按整数年份读取，而不是按日期读取；MySQL 的零年（`0000`）保留为 `0` | +| `YEAR` | `SMALLINT` | 按整数年份读取，而不是按日期读取；MySQL 的零年（`0000`）保留为 `0` | | `FLOAT` | `FLOAT` | | | `DOUBLE` | `DOUBLE` | | | `DECIMAL(p,s)` | `DECIMAL(p,s)` | 超高精度回退为 `STRING` | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md index a5b27a2e0cb7d4..42c5859718c6c0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md @@ -7,16 +7,16 @@ } --- - - -OceanBase CDC 持续导入仅支持 MySQL 兼容模式，并复用 MySQL 的类型转换逻辑。下表展示自动建表时 OceanBase 列类型与 Doris 列类型之间的映射。 - :::caution 实验性功能 -OceanBase CDC 持续导入自 Doris 4.1.4 起作为实验性功能提供。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 ::: + + +OceanBase CDC 持续导入仅支持 MySQL 兼容模式，并复用 MySQL 的类型转换逻辑。下表展示自动建表时 OceanBase 列类型与 Doris 列类型之间的映射。 + ## JDBC URL 参数归一化 创建 OceanBase Streaming Job 时，如果没有显式指定，Doris 会自动在 `jdbc_url` 中追加以下参数： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md index 95adc0222d0905..47653939df457b 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md @@ -6,6 +6,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 下表展示上游 PostgreSQL 的每个列类型如何映射为 Doris 列类型。**只有一套映射**，两种同步方式共用，区别只在于由谁创建目标表： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md index ebdd7caf2e3352..d7bc1a7b759a20 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md index 346207c678f6fa..fd7fdee96985b8 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -143,18 +149,6 @@ SHOW rds.logical_replication; -### Doris 4.0 - -需要预先创建名为 `dbz_publication` 的 Publication，且必须使用 `FOR ALL TABLES`： - -```sql -CREATE PUBLICATION dbz_publication FOR ALL TABLES; -``` - -如果同步用户拥有 superuser 权限（如 `rds_superuser` 角色），Doris 可以在任务启动时自动创建 `dbz_publication`，无需手动执行该语句。Doris 4.0 不支持自定义 Publication 名称或仅指定部分表。 - -### Doris 4.1.0 及之后版本 - 默认情况下，无需手动创建 Publication。未设置 `publication_name` 时，Doris 会使用同步账号创建名为 `doris_pub_` 的 Publication，并在删除作业时清理，因此同步账号需要具有创建 Publication 的权限。 如果在创建 Streaming Job 时显式设置 `publication_name`，则需要预先创建同名 Publication。自定义 Publication 必须包含作业同步的全部源表，并由用户负责维护和清理。例如： @@ -180,11 +174,11 @@ FOR TABLE public.orders, public.customers; **Q3：可以使用自定义名称的 Publication 吗？** -Doris 4.0 不支持，只能使用 `dbz_publication FOR ALL TABLES`。自 4.1.0 版本起，可以通过 `publication_name` 指定自定义名称，但必须在创建作业前创建该 Publication，并确保其包含所有同步表。 +可以通过 `publication_name` 指定自定义名称，但必须在创建作业前创建该 Publication，并确保其包含所有同步表。 **Q4：必须使用 superuser 账号同步吗？** -不必须。Doris 4.0 中，普通同步用户需要手动创建 `dbz_publication`，拥有 superuser 权限时可由 Doris 自动创建。自 4.1.0 版本起，省略 `publication_name` 时 Doris 会自动创建和管理默认 Publication；只有使用自定义 Publication 时才需要预先创建。 +不必须。省略 `publication_name` 时 Doris 会自动创建和管理默认 Publication；只有使用自定义 Publication 时才需要预先创建。 ## Troubleshooting @@ -196,6 +190,5 @@ Doris 4.0 不支持，只能使用 `dbz_publication FOR ALL TABLES`。自 4.1.0 | `SHOW rds.logical_replication;` 仍返回 `off` | 未将新参数组绑定集群，或未重启写入实例 | 确认 **DB cluster parameter group** 已切换为新参数组，并重启写入实例 | | 创建用户或授权失败 | 当前登录账号权限不足 | 使用具备 `rds_superuser` 或等价权限的账号登录后操作 | | `GRANT rds_replication` 报错 | 当前 Aurora 版本过低或集群非 Aurora PostgreSQL | 升级到 Aurora PostgreSQL 14 及以上版本 | -| Doris 4.0 报 `dbz_publication` 不存在 | 未创建 `dbz_publication`，且同步账号无 superuser 权限 | 执行 `CREATE PUBLICATION dbz_publication FOR ALL TABLES;`，或使用具有相应权限的账号 | -| Doris 4.1.0 及之后版本创建默认 Publication 失败 | 同步账号缺少创建 Publication 的权限 | 为同步账号授予所需权限，或由管理员预先创建自定义 Publication 并配置 `publication_name` | -| Doris 4.1.0 及之后版本报自定义 Publication 不存在 | 配置了 `publication_name`，但未预先创建同名 Publication | 按[步骤五](#步骤五配置-publication)创建 Publication，或删除 `publication_name` 配置 | +| 创建默认 Publication 失败 | 同步账号缺少创建 Publication 的权限 | 为同步账号授予所需权限，或由管理员预先创建自定义 Publication 并配置 `publication_name` | +| 自定义 Publication 不存在 | 配置了 `publication_name`，但未预先创建同名 Publication | 按[步骤五](#步骤五配置-publication)创建 Publication，或删除 `publication_name` 配置 | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md index 008a4e2bab6534..aa7248a8bc4c8b 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md index 4b4e4e7e81359c..570881d7982281 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md @@ -18,6 +18,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -160,18 +166,6 @@ Amazon RDS PostgreSQL 使用 `rds_replication` 角色来授予复制权限，而 -### Doris 4.0 - -需要预先创建名为 `dbz_publication` 的 Publication，且必须使用 `FOR ALL TABLES`： - -```sql -CREATE PUBLICATION dbz_publication FOR ALL TABLES; -``` - -如果同步用户拥有 superuser 权限（如 `rds_superuser` 角色），Doris 可以在任务启动时自动创建 `dbz_publication`，无需手动执行该语句。Doris 4.0 不支持自定义 Publication 名称或仅指定部分表。 - -### Doris 4.1.0 及之后版本 - 默认情况下，无需手动创建 Publication。未设置 `publication_name` 时，Doris 会使用同步账号创建名为 `doris_pub_` 的 Publication，并在删除作业时清理，因此同步账号需要具有创建 Publication 的权限。 如果在创建 Streaming Job 时显式设置 `publication_name`，则需要预先创建同名 Publication。自定义 Publication 必须包含作业同步的全部源表，并由用户负责维护和清理。例如： @@ -195,7 +189,7 @@ FOR TABLE public.orders, public.customers; **Q2：可以使用其他名称的 Publication 吗？** -Doris 4.0 不支持，只能使用 `dbz_publication FOR ALL TABLES`。自 4.1.0 版本起，可以通过 `publication_name` 指定自定义名称，但必须在创建作业前创建该 Publication，并确保其包含所有同步表。 +可以通过 `publication_name` 指定自定义名称，但必须在创建作业前创建该 Publication，并确保其包含所有同步表。 **Q3：是否必须使用 `rds_replication` 角色？** @@ -203,7 +197,7 @@ Doris 4.0 不支持，只能使用 `dbz_publication FOR ALL TABLES`。自 4.1.0 **Q4：是否一定要手动创建 Publication？** -Doris 4.0 中，普通同步用户需要手动创建 `dbz_publication`；拥有 superuser 权限时可由 Doris 自动创建。自 4.1.0 版本起，省略 `publication_name` 时 Doris 会自动创建和管理默认 Publication，仅使用自定义 Publication 时需要预先创建。详见[步骤五](#步骤五配置-publication)。 +省略 `publication_name` 时 Doris 会自动创建和管理默认 Publication，仅使用自定义 Publication 时需要预先创建。详见[步骤五](#步骤五配置-publication)。 --- @@ -216,7 +210,6 @@ Doris 4.0 中，普通同步用户需要手动创建 `dbz_publication`；拥有 | `SHOW rds.logical_replication;` 仍为 `off` | 参数组未绑定到实例 / 未重启 | 检查实例是否使用了新的参数组，并执行重启 | | 创建用户报权限错误 | 当前账号权限不足 | 使用具有创建用户权限的管理员账号执行 | | `GRANT rds_replication` 报错 | 当前账号无权授予该角色 | 使用 `rds_superuser` 账号或具备授予权限的账号执行 | -| Doris 4.0 报 `dbz_publication` 不存在 | 未创建 `dbz_publication`，且同步账号无 superuser 权限 | 执行 `CREATE PUBLICATION dbz_publication FOR ALL TABLES;`，或使用具有相应权限的账号 | -| Doris 4.1.0 及之后版本创建默认 Publication 失败 | 同步账号缺少创建 Publication 的权限 | 为同步账号授予所需权限，或由管理员预先创建自定义 Publication 并配置 `publication_name` | -| Doris 4.1.0 及之后版本报自定义 Publication 不存在 | 配置了 `publication_name`，但未预先创建同名 Publication | 按[步骤五](#步骤五配置-publication)创建 Publication，或删除 `publication_name` 配置以使用 Doris 自动管理的默认 Publication | +| 创建默认 Publication 失败 | 同步账号缺少创建 Publication 的权限 | 为同步账号授予所需权限，或由管理员预先创建自定义 Publication 并配置 `publication_name` | +| 自定义 Publication 不存在 | 配置了 `publication_name`，但未预先创建同名 Publication | 按[步骤五](#步骤五配置-publication)创建 Publication，或删除 `publication_name` 配置以使用 Doris 自动管理的默认 Publication | | 修改参数后未生效 | 未选择 **Apply immediately** 或未重启 | 重新执行 [步骤三](#步骤三应用参数组并重启) | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-mysql.md index 1f29025c436daf..fad68e62e75cb0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-mysql.md @@ -16,16 +16,16 @@ } --- - - +:::caution 实验性功能 -MySQL Schema Change 同步用于在持续导入期间，将上游 MySQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md)，不适用于 [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md)。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 -:::tip +::: -MySQL Schema Change 同步自 Doris 4.1.4 起支持。 + + -::: +MySQL Schema Change 同步用于在持续导入期间，将上游 MySQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md)，不适用于 [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md)。 ## 支持的 Schema Change diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md index 6d3ec123372828..92f64a7e3bef10 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md @@ -15,16 +15,18 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + OceanBase Schema Change 同步用于在持续导入期间，将上游 OceanBase 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md)。 -:::caution 实验性功能 - -OceanBase Schema Change 同步自 Doris 4.1.4 起作为实验性功能提供，仅支持 MySQL 兼容模式。 - -::: +OceanBase Schema Change 同步仅支持 MySQL 兼容模式。 ## 支持的 Schema Change diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-postgresql.md index 86bc4c6f6e1fa8..94637264873b46 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/data-operate/import/import-way/streaming-job/schema-change-postgresql.md @@ -16,16 +16,16 @@ } --- - - +:::caution 实验性功能 -PostgreSQL Schema Change 同步用于在持续导入期间，将上游 PostgreSQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md)，不适用于 [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md)。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 -:::tip +::: -PostgreSQL Schema Change 同步自 Doris 4.1.0 起支持。 + + -::: +PostgreSQL Schema Change 同步用于在持续导入期间，将上游 PostgreSQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md)，不适用于 [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md)。 ## 支持的 Schema Change @@ -41,7 +41,7 @@ PostgreSQL Schema Change 同步自 Doris 4.1.0 起支持。 | DEFAULT、`NULL` / `NOT NULL` 及其他约束变化 | 不同步，需自行确认目标表兼容性。 | | 索引及其他表级操作 | 不同步；`TRUNCATE`、`DROP TABLE` 不会清空或删除 Doris 表。 | -自 Doris 4.1.4 起，通过 Relation 事件检测表结构变化。单独执行 DDL 不会立即同步，需等待该表后续的 INSERT、UPDATE 或 DELETE。请在全量同步完成并进入增量同步后，再变更源表结构。 +通过 Relation 事件检测表结构变化。单独执行 DDL 不会立即同步，需等待该表后续的 INSERT、UPDATE 或 DELETE。请在全量同步完成并进入增量同步后，再变更源表结构。 新增列不复制默认值和非空约束，也不回填 Doris 中已有的数据。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x.json b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x.json index efd040ad9d0641..7584b5e2a3bd88 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x.json +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x.json @@ -723,17 +723,17 @@ "message": "更多", "description": "The label for category More in sidebar docs" }, - "sidebar.docs.category.MySQL (Experimental)": { - "message": "MySQL（实验性）", - "description": "The label for category MySQL (Experimental) in sidebar docs" + "sidebar.docs.category.MySQL": { + "message": "MySQL", + "description": "The label for category MySQL in sidebar docs" }, "sidebar.docs.category.Optimization Technology Principles": { "message": "优化技术原理", "description": "The label for category Optimization Technology Principles in sidebar docs" }, - "sidebar.docs.category.OceanBase (Experimental)": { - "message": "OceanBase（实验性）", - "description": "The label for category OceanBase (Experimental) in sidebar docs" + "sidebar.docs.category.OceanBase": { + "message": "OceanBase", + "description": "The label for category OceanBase in sidebar docs" }, "sidebar.docs.category.Paimon Catalog": { "message": "Paimon Catalog", @@ -743,9 +743,9 @@ "message": "性能优化", "description": "The label for category Performance in sidebar docs" }, - "sidebar.docs.category.PostgreSQL (Experimental)": { - "message": "PostgreSQL（实验性）", - "description": "The label for category PostgreSQL (Experimental) in sidebar docs" + "sidebar.docs.category.PostgreSQL": { + "message": "PostgreSQL", + "description": "The label for category PostgreSQL in sidebar docs" }, "sidebar.docs.category.Query Performance": { "message": "查询性能", diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md index 43d82cc3cc8950..9a2273859cf147 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -146,7 +152,7 @@ MySQL 源端 (`FROM MYSQL`) 支持的参数如下： | include_tables | - | 需要同步的表名，多个表用逗号分隔，不填默认所有的表 | | offset | latest | `latest`：仅增量同步；`initial`：全量 + 增量同步 | | `schema_change_behavior` | `evolve` | 可选。 字符串，不区分大小写。`evolve` 自动同步支持的列变更，`ignore` 关闭结构同步。可暂停后通过 `ALTER JOB` 修改；支持范围及暂停处理见 [Schema Change 同步](./schema-change-mysql.md)。 | -| snapshot_split_size | 40960（4.1.4 之前为 8096） | split 的大小（行数），全量同步时，表会被切分成多个 split 进行同步 | +| snapshot_split_size | 40960 | split 的大小（行数），全量同步时，表会被切分成多个 split 进行同步 | | snapshot_parallelism | 1 | 全量阶段同步的并行度，即单次 Task 最多调度的 split 数量 | ## 参考手册 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md index 9dc3ab2a81f24c..5fbe01e00a2022 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -34,7 +40,7 @@ SQL 映射同步通过 Job + [CDC Stream TVF](../../../../sql-manual/sql-functio | 检查项 | 说明 | | ------ | ---- | -| Doris 版本 | 4.1.0 及以上 | +| Doris 版本 | 4.1.5 及以上 | | 表类型 | 目前仅支持**主键表**作为目标表 | | 用户权限 | 需要 `Load` 权限 | | MySQL 配置 | 需要在 MySQL 端开启 Binlog，参考[配置指南](./continuous-load-overview.md) | @@ -93,7 +99,7 @@ CDC Stream TVF 支持的 MySQL 数据源参数如下： | database | - | 数据库名 | | table | - | 需要同步的表名 | | offset | latest | `latest`：仅增量同步；`initial`：全量 + 增量同步 | -| snapshot_split_size | 40960（4.1.4 之前为 8096） | split 的大小（行数）。全量同步时，表会被切分成多个 split 进行同步 | +| snapshot_split_size | 40960 | split 的大小（行数）。全量同步时，表会被切分成多个 split 进行同步 | | snapshot_parallelism | 1 | 全量阶段同步的并行度，即单次 Task 最多调度的 split 数量 | ### 导入配置参数 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md index 7d6fc5704e91fa..e58fdf36b42ee8 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md @@ -16,16 +16,18 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + OceanBase CDC 自动建表同步通过 `FROM OCEANBASE (...) TO DATABASE (...)` 实现。Doris 从 OBBinlog 服务读取 OceanBase 的全量和增量数据，并根据上游表结构自动创建 Doris 主键表。可以通过 `include_tables` 同步一张、多张或全部表。 -:::caution 实验性功能 - -OceanBase CDC 持续导入自 Doris 4.1.4 起作为实验性功能提供。目前仅支持 OceanBase 的 MySQL 兼容模式和自动建表同步，不支持 Oracle 兼容模式与 SQL 映射同步。 - -::: +目前仅支持 OceanBase 的 MySQL 兼容模式和自动建表同步，不支持 Oracle 兼容模式与 SQL 映射同步。 ### 适用场景 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md index c6318d7c86c3e2..c9131e8ac3e3db 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md @@ -20,15 +20,17 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + Doris 支持通过 **Streaming Job** 的方式，从多种数据源持续导入数据到 Doris 表中。提交 Job 后，Doris 会持续运行导入作业，实时读取数据源中的数据并写入到 Doris 表中。 -:::tip -该功能自 4.1.0 版本起支持。 -::: - 本文将帮助你解决以下问题： - 持续导入支持哪些数据源和同步模式？ @@ -39,13 +41,13 @@ Doris 支持通过 **Streaming Job** 的方式，从多种数据源持续导入 ## 支持的数据源与同步模式 -持续导入支持以下数据源和同步模式： +持续导入支持以下数据源和同步模式。S3 通知导入和一次性导入自 Doris 4.1.6 版本起支持： | 数据源 | 支持版本 | SQL 映射同步 | 自动建表同步 | 配置指南 | | :--------- | :---------------- | :-------------------------------------------------------------------- | :-------------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | | MySQL | 5.6、5.7、8.0.x | [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md) | [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md) | [Amazon RDS MySQL](./prerequisites/amazon-rds-mysql.md) · [Amazon Aurora MySQL](./prerequisites/amazon-aurora-mysql.md) | | PostgreSQL | 14、15、16、17 | [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md) | [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md) | [Amazon RDS PostgreSQL](./prerequisites/amazon-rds-postgresql.md) · [Amazon Aurora PostgreSQL](./prerequisites/amazon-aurora-postgresql.md) | -| OceanBase | MySQL 兼容模式 | - | [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md)（自 4.1.4 版本起） | - | +| OceanBase | MySQL 兼容模式 | - | [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md) | - | | S3 | - | [S3 导入概览](./continuous-load-s3.md) | - | - | 上游列类型如何映射为 Doris 类型，见 [MySQL](./data-type-mapping-mysql.md)、[PostgreSQL](./data-type-mapping-postgresql.md)和 [OceanBase](./data-type-mapping-oceanbase.md) 数据类型映射。 @@ -146,8 +148,8 @@ select * from jobs("type"="insert") where ExecuteType = "STREAMING"; | LoadStatistic | Job 的统计信息 | | ErrorMsg | Job 执行的错误信息 | | JobRuntimeMsg | Job 运行时的一些提示信息 | -| LagBytes | 数据源端日志（MySQL / OceanBase Binlog、PostgreSQL WAL）的积压字节数，`-1` 表示当前不可用（例如 S3 数据源或全量快照阶段）。**自 4.1.4 版本起**，该列由原来的 `Lag`（单位：秒）改名为 `LagBytes`（单位：字节） | -| LastSourceEventTimestamp | 已提交 Offset 中记录的数据源端最新事件时间戳（Unix 秒），为空表示不可用。**自 4.1.4 版本起新增** | +| LagBytes | 数据源端日志（MySQL / OceanBase Binlog、PostgreSQL WAL）的积压字节数，`-1` 表示当前不可用（例如 S3 数据源或全量快照阶段） | +| LastSourceEventTimestamp | 已提交 Offset 中记录的数据源端最新事件时间戳（Unix 秒），为空表示不可用 | | LastTaskSuccessTime | 最近一次 Task 成功完成的时间 | ### 查看 Task 状态 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md index cf1b5a7ad1acd1..d14bd06e026832 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md @@ -19,6 +19,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -97,7 +103,7 @@ PostgreSQL 源端参数用于配置 JDBC 连接、同步范围以及全量切片 | include_tables | - | 需要同步的表名，多个表用逗号分隔，不填默认所有的表 | | offset | latest | latest: 仅增量同步，initial: 全量 + 增量同步 | | `schema_change_behavior` | `evolve` | 可选。 字符串，不区分大小写。`evolve` 自动同步支持的列变更，`ignore` 关闭结构同步。可暂停后通过 `ALTER JOB` 修改；支持范围及暂停处理见 [Schema Change 同步](./schema-change-postgresql.md)。 | -| snapshot_split_size | 40960（4.1.4 之前为 8096） | split 的大小（行数），全量同步时，表会被切分成多个 split 进行同步 | +| snapshot_split_size | 40960 | split 的大小（行数），全量同步时，表会被切分成多个 split 进行同步 | | snapshot_parallelism | 1 | 全量阶段同步的并行度，即单次 Task 最多调度的 split 数量 | ### Doris 目标库端配置参数 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md index 1b3d9a96a59fb0..7e0a7a318f5f83 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + @@ -103,7 +109,7 @@ select * from jobs("type"="insert") where ExecuteType = "STREAMING"; | schema | - | Schema 名称 | | table | - | 需要同步的表名 | | offset | latest | `latest`：仅增量同步；`initial`：全量 + 增量同步 | -| snapshot_split_size | 40960（4.1.4 之前为 8096） | split 的大小（行数）。全量同步时，表会被切分成多个 split 进行同步 | +| snapshot_split_size | 40960 | split 的大小（行数）。全量同步时，表会被切分成多个 split 进行同步 | | snapshot_parallelism | 1 | 全量阶段同步的并行度，即单次 Task 最多调度的 split 数量 | ### 导入配置参数 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md index c2aec48bfa5170..078de88aec2016 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +S3 通知导入为实验性功能，自 Doris 4.1.6 版本起支持。 + +::: + 通知导入通过 `NOTIFICATION` 模式，消费 SQS 中的 S3 对象创建事件，将新文件持续导入 Doris。适用于 UUID 文件名、乱序到达或文件名无法单调递增的场景，无需依赖文件名排序。 目前仅支持 **AWS S3 直接发送到同 Region 的 SQS Standard Queue**。每个作业需要独占队列；只处理队列中的通知，不补导历史对象。端到端采用 at-least-once 语义，可能重复导入同一对象，且不保证文件导入顺序。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md index 1f716824b6b0ac..b055a779656259 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +S3 一次性导入为实验性功能，自 Doris 4.1.6 版本起支持。 + +::: + 一次性导入通过 `ONE_TIME` 模式，将指定 S3 路径中匹配的文件分批导入已有的 Doris 表，最后一批事务提交后作业进入 `FINISHED`。适用于静态历史数据初始化、一次性的离线数据入仓或独立前缀下的回补任务。 创建前应准备好所有文件，导入期间保持源路径稳定。作业不会在完成后持续等待新文件；如需持续接入，使用[有序导入](./continuous-load-s3-ordered.md)或[通知导入](./continuous-load-s3-notification.md)。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md index d2e5a7de03f783..63fb4072844583 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md @@ -12,6 +12,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 有序导入通过 `ORDERED_LIST` 模式，按 S3 对象 key（文件名及路径）的**字典序**持续发现新文件，适用于按时间戳或递增序号命名的日志、埋点及 ETL 文件。这里的“有序”指文件发现规则，不表示目标表中的行具有顺序保证。 这是默认的 S3 导入模式，无需配置消息队列。若文件可能乱序到达，使用[通知导入](./continuous-load-s3-notification.md)；若只需导入一批已准备好的文件，使用[一次性导入](./continuous-load-s3-one-time.md)。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md index c6a6fef627eee7..de5481bb4fed8a 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md @@ -12,7 +12,13 @@ } --- -Doris 通过 **Streaming Job + S3 TVF** 将 S3 文件分批导入已有的 Doris 表。根据文件生成方式和是否需要持续运行，选择有序导入、通知导入或一次性导入。 +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + +Doris 通过 **Streaming Job + S3 TVF** 将 S3 文件分批导入已有的 Doris 表。根据文件生成方式和是否需要持续运行，选择有序导入、通知导入或一次性导入。通知导入和一次性导入自 Doris 4.1.6 版本起支持。 ## 选择导入模式 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md index bbd03480db5e43..f9eca56ff40f1e 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md @@ -6,6 +6,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 下表展示上游 MySQL 的每个列类型如何映射为 Doris 列类型。**只有一套映射**，两种同步方式共用，区别只在于由谁创建目标表： @@ -17,8 +23,6 @@ ## JDBC URL 参数归一化 -> 自 4.1.4 版本开始生效。 - 创建 MySQL 数据源的 Streaming Job（以及使用 `cdc_stream()` 表函数）时，如果用户没有显式指定，Doris 会自动在 `jdbc_url` 上追加以下参数： | 参数 | 注入值 | 影响 | @@ -32,7 +36,7 @@ :::caution 升级影响 -从 4.1.3 及更早版本升级后，已有的 MySQL CDC 作业中 `TINYINT(1)` 列的取值展示会由 `true` / `false` 变为 `1` / `0`，`YEAR` 列会由日期变为整数年份。如果需要保留旧行为，可在 `jdbc_url` 中显式设置 `tinyInt1isBit=true` 和 `yearIsDateType=true`。 +如需将 `TINYINT(1)` 按 `BOOLEAN` 读取，或将 `YEAR` 按日期读取，可在 `jdbc_url` 中分别显式设置 `tinyInt1isBit=true` 和 `yearIsDateType=true`。 ::: @@ -40,13 +44,13 @@ | MySQL 类型 | Doris 类型 | 备注 | | --- | --- | --- | -| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | **自 4.1.4 版本起**映射为 `TINYINT`（取值 `1` / `0`）；4.1.4 之前映射为 `BOOLEAN`（`true` / `false`）。原因见下方【JDBC URL 参数归一化】 | +| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | 映射为 `TINYINT`（取值 `1` / `0`）。原因见下方【JDBC URL 参数归一化】 | | `TINYINT` | `TINYINT` | `UNSIGNED` → `SMALLINT` | | `SMALLINT` | `SMALLINT` | `UNSIGNED` → `INT` | | `MEDIUMINT` | `INT` | `UNSIGNED` → `INT` | | `INT` | `INT` | `UNSIGNED` → `BIGINT` | | `BIGINT` | `BIGINT` | `UNSIGNED` → `LARGEINT` | -| `YEAR` | `SMALLINT` | **自 4.1.4 版本起**按整数年份读取，而不是按日期读取；MySQL 的零年（`0000`）保留为 `0` | +| `YEAR` | `SMALLINT` | 按整数年份读取，而不是按日期读取；MySQL 的零年（`0000`）保留为 `0` | | `FLOAT` | `FLOAT` | | | `DOUBLE` | `DOUBLE` | | | `DECIMAL(p,s)` | `DECIMAL(p,s)` | 超高精度回退为 `STRING` | diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md index a5b27a2e0cb7d4..42c5859718c6c0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md @@ -7,16 +7,16 @@ } --- - - -OceanBase CDC 持续导入仅支持 MySQL 兼容模式，并复用 MySQL 的类型转换逻辑。下表展示自动建表时 OceanBase 列类型与 Doris 列类型之间的映射。 - :::caution 实验性功能 -OceanBase CDC 持续导入自 Doris 4.1.4 起作为实验性功能提供。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 ::: + + +OceanBase CDC 持续导入仅支持 MySQL 兼容模式，并复用 MySQL 的类型转换逻辑。下表展示自动建表时 OceanBase 列类型与 Doris 列类型之间的映射。 + ## JDBC URL 参数归一化 创建 OceanBase Streaming Job 时，如果没有显式指定，Doris 会自动在 `jdbc_url` 中追加以下参数： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md index 95adc0222d0905..47653939df457b 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md @@ -6,6 +6,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + 下表展示上游 PostgreSQL 的每个列类型如何映射为 Doris 列类型。**只有一套映射**，两种同步方式共用，区别只在于由谁创建目标表： diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md index ebdd7caf2e3352..d7bc1a7b759a20 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md index dc6e3b8af5a90c..7b53c9012900cc 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md @@ -16,6 +16,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md index 008a4e2bab6534..aa7248a8bc4c8b 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md index 05a718a93da395..82a43ca6436269 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md @@ -17,6 +17,12 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md index 1f29025c436daf..fad68e62e75cb0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md @@ -16,16 +16,16 @@ } --- - - +:::caution 实验性功能 -MySQL Schema Change 同步用于在持续导入期间，将上游 MySQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md)，不适用于 [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md)。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 -:::tip +::: -MySQL Schema Change 同步自 Doris 4.1.4 起支持。 + + -::: +MySQL Schema Change 同步用于在持续导入期间，将上游 MySQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [MySQL CDC 自动建表同步](./continuous-load-mysql-database.md)，不适用于 [MySQL CDC SQL 映射同步](./continuous-load-mysql-table.md)。 ## 支持的 Schema Change diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md index 6d3ec123372828..92f64a7e3bef10 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md @@ -15,16 +15,18 @@ } --- +:::caution 实验性功能 + +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 + +::: + OceanBase Schema Change 同步用于在持续导入期间，将上游 OceanBase 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [OceanBase CDC 自动建表同步](./continuous-load-oceanbase-database.md)。 -:::caution 实验性功能 - -OceanBase Schema Change 同步自 Doris 4.1.4 起作为实验性功能提供，仅支持 MySQL 兼容模式。 - -::: +OceanBase Schema Change 同步仅支持 MySQL 兼容模式。 ## 支持的 Schema Change diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md index 44625ee4faa819..94637264873b46 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md @@ -16,16 +16,16 @@ } --- - - +:::caution 实验性功能 -PostgreSQL Schema Change 同步用于在持续导入期间，将上游 PostgreSQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md)，不适用于 [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md)。 +Doris 持续导入为实验性功能，自 Doris 4.1.5 版本起支持。 -:::tip +::: -PostgreSQL Schema Change 同步自 Doris 4.1.0 起支持。 + + -::: +PostgreSQL Schema Change 同步用于在持续导入期间，将上游 PostgreSQL 表的列结构变更自动应用到 Doris 目标表。该能力仅适用于 [PostgreSQL CDC 自动建表同步](./continuous-load-postgresql-database.md)，不适用于 [PostgreSQL CDC SQL 映射同步](./continuous-load-postgresql-table.md)。 ## 支持的 Schema Change @@ -41,9 +41,9 @@ PostgreSQL Schema Change 同步自 Doris 4.1.0 起支持。 | DEFAULT、`NULL` / `NOT NULL` 及其他约束变化 | 不同步，需自行确认目标表兼容性。 | | 索引及其他表级操作 | 不同步；`TRUNCATE`、`DROP TABLE` 不会清空或删除 Doris 表。 | -自 Doris 4.1.4 起，通过 Relation 事件检测表结构变化。单独执行 DDL 不会立即同步，需等待该表后续的 INSERT、UPDATE 或 DELETE。请在全量同步完成并进入增量同步后，再变更源表结构。 +通过 Relation 事件检测表结构变化。单独执行 DDL 不会立即同步，需等待该表后续的 INSERT、UPDATE 或 DELETE。请在全量同步完成并进入增量同步后，再变更源表结构。 -自 Doris 4.1.4 起，新增列不复制默认值和非空约束，也不回填 Doris 中已有的数据。 +新增列不复制默认值和非空约束，也不回填 Doris 中已有的数据。 ## 配置 Schema Change 行为 diff --git a/sidebars.ts b/sidebars.ts index bd8783b5cdb73a..19d6b10cc80daf 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -389,7 +389,7 @@ const sidebars: SidebarsConfig = { items: [ { type: 'category', - label: 'MySQL (Experimental)', + label: 'MySQL', items: [ 'data-operate/import/import-way/streaming-job/continuous-load-mysql-table', 'data-operate/import/import-way/streaming-job/continuous-load-mysql-database', @@ -407,7 +407,7 @@ const sidebars: SidebarsConfig = { }, { type: 'category', - label: 'PostgreSQL (Experimental)', + label: 'PostgreSQL', items: [ 'data-operate/import/import-way/streaming-job/continuous-load-postgresql-table', 'data-operate/import/import-way/streaming-job/continuous-load-postgresql-database', @@ -425,7 +425,7 @@ const sidebars: SidebarsConfig = { }, { type: 'category', - label: 'OceanBase (Experimental)', + label: 'OceanBase', items: [ 'data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database', 'data-operate/import/import-way/streaming-job/schema-change-oceanbase', diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md index c06dbeee7370de..1aac80b895a2c7 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-database.md @@ -17,6 +17,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -145,7 +151,7 @@ The MySQL source side (`FROM MYSQL`) supports the following parameters: | database | - | Database name | | include_tables | - | Names of tables to sync, separated by commas. If left empty, all tables are synced by default | | offset | latest | `latest`: incremental sync only; `initial`: full + incremental sync | -| snapshot_split_size | 40960 (8096 before 4.1.4) | Size of a split (in rows). During full sync, a table is divided into multiple splits for sync | +| snapshot_split_size | 40960 | Size of a split (in rows). During full sync, a table is divided into multiple splits for sync | | snapshot_parallelism | 1 | Parallelism during the full-sync phase, that is, the maximum number of splits a single Task can schedule at a time | ## Reference Manual diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md index 4191e69a16beec..4a1667c489b2ee 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-mysql-table.md @@ -17,6 +17,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -34,7 +40,7 @@ By integrating the read capability of [Flink CDC](https://github.com/apache/flin | Item | Description | | ------ | ---- | -| Doris version | 4.1.0 and above | +| Doris version | 4.1.5 and above | | Table type | Currently only **primary key tables** are supported as the target table | | User privileges | `Load` privilege is required | | MySQL configuration | Binlog must be enabled on the MySQL side. See the [Configuration Guide](./continuous-load-overview.md) | @@ -93,7 +99,7 @@ The MySQL data source parameters supported by CDC Stream TVF are as follows: | database | - | Database name | | table | - | Name of the table to sync | | offset | latest | `latest`: incremental sync only; `initial`: full + incremental sync | -| snapshot_split_size | 40960 (8096 before 4.1.4) | Size of a split (in rows). During full sync, the table is divided into multiple splits for syncing | +| snapshot_split_size | 40960 | Size of a split (in rows). During full sync, the table is divided into multiple splits for syncing | | snapshot_parallelism | 1 | Parallelism during the full sync stage, that is, the maximum number of splits scheduled per Task | ### Import Configuration Parameters diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md index 0a8015393a78c4..6723db3500c7dd 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database.md @@ -16,16 +16,18 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + OceanBase CDC with Auto Table Creation uses `FROM OCEANBASE (...) TO DATABASE (...)`. Doris reads full and incremental data through the OBBinlog service and automatically creates Doris Unique Key tables from the upstream schemas. Use `include_tables` to sync one table, multiple tables, or all tables in the database. -:::caution Experimental feature - -OceanBase CDC continuous load is available as an experimental feature starting from Doris 4.1.4. It currently supports only OceanBase MySQL compatibility mode and Auto Table Creation Sync. Oracle compatibility mode and SQL Mapping Sync are not supported. - -::: +OceanBase CDC continuous load currently supports only OceanBase MySQL compatibility mode and Auto Table Creation Sync. Oracle compatibility mode and SQL Mapping Sync are not supported. ### Applicable Scenarios diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md index d2a63e8054b927..ce0cdf257b6230 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-overview.md @@ -20,15 +20,17 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + Doris supports continuously loading data from multiple data sources into Doris tables through **Streaming Job**. After a Job is submitted, Doris keeps the load job running, reading data from the source in real time and writing it into the Doris table. -:::tip -This feature is supported starting from version 4.1.0. -::: - This document helps you answer the following questions: - Which data sources and sync modes does continuous load support? @@ -39,13 +41,13 @@ This document helps you answer the following questions: ## Supported Data Sources and Sync Modes -Continuous load supports the following data sources and sync modes: +Continuous load supports the following data sources and sync modes. S3 notification and one-time ingestion are supported starting with Doris 4.1.6: | Data Source | Supported Versions | SQL Mapping Sync | Auto Table Creation Sync | Configuration Guide | | :---------- | :----------------- | :---------------------------------------------------------------- | :---------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | | MySQL | 5.6, 5.7, 8.0.x | [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md) | [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md) | [Amazon RDS MySQL](./prerequisites/amazon-rds-mysql.md) · [Amazon Aurora MySQL](./prerequisites/amazon-aurora-mysql.md) | | PostgreSQL | 14, 15, 16, 17 | [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md) | [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md) | [Amazon RDS PostgreSQL](./prerequisites/amazon-rds-postgresql.md) · [Amazon Aurora PostgreSQL](./prerequisites/amazon-aurora-postgresql.md) | -| OceanBase | MySQL compatibility mode | - | [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md) (since version 4.1.4) | - | +| OceanBase | MySQL compatibility mode | - | [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md) | - | | S3 | - | [S3 Continuous Load](./continuous-load-s3.md) | - | - | For how upstream column types map to Doris types, see Data Type Mapping for [MySQL](./data-type-mapping-mysql.md), [PostgreSQL](./data-type-mapping-postgresql.md), and [OceanBase](./data-type-mapping-oceanbase.md). @@ -142,8 +144,8 @@ Result columns: | LoadStatistic | Job statistics | | ErrorMsg | Error message of the Job | | JobRuntimeMsg | Runtime hints of the Job | -| LagBytes | Number of backlog bytes in the source log (MySQL / OceanBase binlog or PostgreSQL WAL). `-1` means the value is currently unavailable, for example for an S3 data source or during the full snapshot phase. **Since version 4.1.4**, this column replaces the former `Lag` column (in seconds) and is reported in bytes | -| LastSourceEventTimestamp | Timestamp (in Unix seconds) of the latest source event recorded in the committed offset. Empty when unavailable. **Added in version 4.1.4** | +| LagBytes | Number of backlog bytes in the source log (MySQL / OceanBase binlog or PostgreSQL WAL). `-1` means the value is currently unavailable, for example for an S3 data source or during the full snapshot phase. | +| LastSourceEventTimestamp | Timestamp (in Unix seconds) of the latest source event recorded in the committed offset. Empty when unavailable. | | LastTaskSuccessTime | Time when the most recent Task completed successfully | ### View Task Status @@ -227,16 +229,16 @@ Only upstream tables **with a primary key** can be synchronized (both sync metho DDL sync applies **only to Auto Table Creation Sync**; SQL Mapping (TVF) does not sync any DDL — the `cdc_stream()` table function always forces `schema_change_enabled` to `false`. -- **PostgreSQL** (supported since 4.1): only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and constraint / index / partition changes are NOT synced** — apply them manually in Doris. -- **MySQL** (supported since 4.1.4): only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and constraint / index / partition changes are NOT synced** — apply them manually in Doris. -- **OceanBase** (supported since 4.1.4): only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and primary key / constraint / index / partition changes are NOT synced** — apply them manually in Doris. +- **PostgreSQL**: only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and constraint / index / partition changes are NOT synced** — apply them manually in Doris. +- **MySQL**: only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and constraint / index / partition changes are NOT synced** — apply them manually in Doris. +- **OceanBase**: only `ADD COLUMN` and `DROP COLUMN` are synced. **Column type changes, `RENAME COLUMN`, and primary key / constraint / index / partition changes are NOT synced** — apply them manually in Doris. Auto Table Creation Sync enables Schema Change Sync by default. Doris 4.1 currently does not provide a SQL property to disable it. -:::caution Behavior change (4.1.4) +:::caution Schema change behavior -- Starting from version 4.1.4, an added column **no longer carries the `DEFAULT` value of the upstream column** (for MySQL, PostgreSQL, and OceanBase). The new column has no default value in Doris, and historical rows are not backfilled. -- PostgreSQL schema change detection is now driven by Relation events: only `ADD COLUMN` and `DROP COLUMN` are recognized; a change that both adds and drops columns (possibly a `RENAME`) is skipped, as are column type changes. This capability applies only to the Auto Table Creation Sync (at-least-once) path; the TVF / exactly-once path does not support it. +- An added column **does not carry the `DEFAULT` value of the upstream column** (for MySQL, PostgreSQL, and OceanBase). The new column has no default value in Doris, and historical rows are not backfilled. +- PostgreSQL schema change detection is driven by Relation events: only `ADD COLUMN` and `DROP COLUMN` are recognized; a change that both adds and drops columns (possibly a `RENAME`) is skipped, as are column type changes. This capability applies only to the Auto Table Creation Sync (at-least-once) path; the TVF / exactly-once path does not support it. ::: diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md index da9ce1a1324cc9..6042a1adb22e0a 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-database.md @@ -19,6 +19,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -96,7 +102,7 @@ PostgreSQL source parameters configure the JDBC connection, sync scope, and full | schema | - | Schema name | | include_tables | - | Tables to sync, separated by commas. If left empty, all tables are synced by default | | offset | latest | latest: incremental-only sync; initial: full + incremental sync | -| snapshot_split_size | 40960 (8096 before 4.1.4) | Size of a split (in rows). During full sync, a table is divided into multiple splits for synchronization | +| snapshot_split_size | 40960 | Size of a split (in rows). During full sync, a table is divided into multiple splits for synchronization | | snapshot_parallelism | 1 | Parallelism during the full-sync phase, that is, the maximum number of splits scheduled in a single Task | ### Doris Target Database Parameters diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md index 99e1b7b9d0d66d..3f7ecec80ea7ac 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-postgresql-table.md @@ -17,6 +17,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + @@ -103,7 +109,7 @@ Data source parameters are configured through the `cdc_stream(...)` TVF. They de | schema | - | Schema name. | | table | - | Name of the table to sync. | | offset | latest | `latest`: incremental sync only. `initial`: full + incremental sync. | -| snapshot_split_size | 40960 (8096 before 4.1.4) | Split size in rows. During full sync, the table is divided into multiple splits for syncing. | +| snapshot_split_size | 40960 | Split size in rows. During full sync, the table is divided into multiple splits for syncing. | | snapshot_parallelism | 1 | Parallelism for the full sync stage, that is, the maximum number of splits scheduled per Task. | ### Load Configuration Parameters diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md index 94f56189f2d6e4..fbee49706bbbaa 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-notification.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +S3 notification ingestion is experimental and supported starting with Doris 4.1.6. + +::: + Notification ingestion uses `NOTIFICATION` to consume S3 object-created events from SQS and continuously ingest new files into Doris. It suits UUID filenames and files arriving in any order, without relying on filename ordering. Only **AWS S3 notifications sent directly to an SQS Standard Queue in the same Region** are supported. Each job needs a dedicated queue. Historical objects are not scanned or backfilled. End-to-end delivery is at-least-once: an object can be ingested more than once, and file ingestion order is not guaranteed. diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md index 77f33f1f6f57e0..4f330a4724c132 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-one-time.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +S3 one-time ingestion is experimental and supported starting with Doris 4.1.6. + +::: + One-time ingestion uses `ONE_TIME` to ingest matching S3 files into an existing Doris table in batches. The job enters `FINISHED` after the final batch commits. It suits historical initialization, a one-off offline load, or a backfill under a separate prefix. Prepare all files before creation and keep the source path stable during ingestion. The completed job does not wait for future files. For continuous ingestion, choose [ordered ingestion](./continuous-load-s3-ordered.md) or [notification ingestion](./continuous-load-s3-notification.md). diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md index 1c47200f75918a..78132515f8a4bd 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3-ordered.md @@ -12,6 +12,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + Ordered ingestion uses `ORDERED_LIST` to continuously discover new S3 objects in **lexicographical key order**, including the path and filename. It suits logs and ETL files named with increasing timestamps or sequence numbers. This order describes file discovery and does not guarantee row order in the target table. This is the default S3 ingestion mode and requires no queue. For files arriving in any order, use [notification ingestion](./continuous-load-s3-notification.md). For a prepared static file set, use [one-time ingestion](./continuous-load-s3-one-time.md). diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md index 04e8de662c1f8f..936d05dce30d54 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/continuous-load-s3.md @@ -12,7 +12,13 @@ } --- -Doris uses **Streaming Job + S3 TVF** to ingest S3 files into an existing Doris table in batches. Choose ordered, notification, or one-time ingestion based on how files arrive and whether the job should continue running. +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + +Doris uses **Streaming Job + S3 TVF** to ingest S3 files into an existing Doris table in batches. Choose ordered, notification, or one-time ingestion based on how files arrive and whether the job should continue running. Notification and one-time ingestion are supported starting with Doris 4.1.6. ## Choose an ingestion mode diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md index 38b69074742461..d003c0256a473f 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-mysql.md @@ -6,6 +6,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + The table below shows how each upstream MySQL column type maps to a Doris column type. There is a single mapping, shared by both sync methods — the only difference is who creates the target table: @@ -17,8 +23,6 @@ In both cases, the values are written following the same mapping. For types that ## JDBC URL Parameter Normalization -> Supported since version 4.1.4. - When you create a Streaming Job with a MySQL data source (or use the `cdc_stream()` table function), Doris automatically appends the following parameters to `jdbc_url` unless you set them explicitly: | Parameter | Injected Value | Effect | @@ -32,7 +36,7 @@ If any of these parameters is set explicitly in `jdbc_url`, Doris keeps your val :::caution Upgrade impact -After upgrading from 4.1.3 or earlier, `TINYINT(1)` columns in existing MySQL CDC jobs are reported as `1` / `0` instead of `true` / `false`, and `YEAR` columns as integer years instead of dates. To keep the previous behavior, set `tinyInt1isBit=true` and `yearIsDateType=true` explicitly in `jdbc_url`. +To read `TINYINT(1)` as `BOOLEAN` or `YEAR` as a date, explicitly set `tinyInt1isBit=true` or `yearIsDateType=true`, respectively, in `jdbc_url`. ::: @@ -40,13 +44,13 @@ After upgrading from 4.1.3 or earlier, `TINYINT(1)` columns in existing MySQL CD | MySQL Type | Doris Type | Notes | | --- | --- | --- | -| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | **Since version 4.1.4** this maps to `TINYINT` (values `1` / `0`); before 4.1.4 it mapped to `BOOLEAN` (`true` / `false`). See [JDBC URL Parameter Normalization](#jdbc-url-parameter-normalization) for the reason | +| `BOOLEAN` / `TINYINT(1)` | `TINYINT` | Maps to `TINYINT` (values `1` / `0`). See [JDBC URL Parameter Normalization](#jdbc-url-parameter-normalization) for the reason | | `TINYINT` | `TINYINT` | `UNSIGNED` → `SMALLINT` | | `SMALLINT` | `SMALLINT` | `UNSIGNED` → `INT` | | `MEDIUMINT` | `INT` | `UNSIGNED` → `INT` | | `INT` | `INT` | `UNSIGNED` → `BIGINT` | | `BIGINT` | `BIGINT` | `UNSIGNED` → `LARGEINT` | -| `YEAR` | `SMALLINT` | **Since version 4.1.4** the value is read as an integer year rather than as a date; the MySQL zero year (`0000`) is kept as `0` | +| `YEAR` | `SMALLINT` | The value is read as an integer year rather than as a date; the MySQL zero year (`0000`) is kept as `0` | | `FLOAT` | `FLOAT` | | | `DOUBLE` | `DOUBLE` | | | `DECIMAL(p,s)` | `DECIMAL(p,s)` | Very high precision falls back to `STRING` | diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md index 16ea4b15cf2f1e..6d449674101f25 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-oceanbase.md @@ -7,16 +7,16 @@ } --- - - -OceanBase CDC continuous load supports only MySQL compatibility mode and uses the MySQL type conversion path. The following table shows how OceanBase column types map to Doris column types during automatic table creation. - :::caution Experimental feature -OceanBase CDC continuous load is available as an experimental feature starting from Doris 4.1.4. +Doris continuous load is experimental and supported starting with Doris 4.1.5. ::: + + +OceanBase CDC continuous load supports only MySQL compatibility mode and uses the MySQL type conversion path. The following table shows how OceanBase column types map to Doris column types during automatic table creation. + ## JDBC URL Parameter Normalization When you create an OceanBase Streaming Job, Doris automatically appends the following parameters to `jdbc_url` unless you set them explicitly: diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md index bf89b8b68fdf8b..cf7ececf50b509 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/data-type-mapping-postgresql.md @@ -6,6 +6,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + The table below shows how each upstream PostgreSQL column type maps to a Doris column type. There is a single mapping, shared by both sync methods — the only difference is who creates the target table: diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md index 715d55dbe79e36..6f9e3939aa7f88 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md index 25f49e4bb50e88..9dfa2f7f565395 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-aurora-postgresql.md @@ -15,6 +15,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md index 3cefe58c4d4a5c..e628df91b71032 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-mysql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md index 5b79cedbf1e67c..07ce3d00a06779 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/prerequisites/amazon-rds-postgresql.md @@ -16,6 +16,12 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md index ae10801b3f5d82..3f6c3a7cf4afc3 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-mysql.md @@ -16,16 +16,16 @@ } --- - - +:::caution Experimental feature -MySQL Schema Change Sync automatically applies upstream MySQL column changes to Doris during continuous load. This capability applies only to [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md). It is not supported by [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md). +Doris continuous load is experimental and supported starting with Doris 4.1.5. -:::tip +::: -MySQL Schema Change Sync is supported from Doris 4.1.4. + + -::: +MySQL Schema Change Sync automatically applies upstream MySQL column changes to Doris during continuous load. This capability applies only to [MySQL CDC with Auto Table Creation](./continuous-load-mysql-database.md). It is not supported by [MySQL CDC with SQL Mapping](./continuous-load-mysql-table.md). ## Supported Schema Changes diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md index 958b3be3531cc0..3bd25cba8dd081 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-oceanbase.md @@ -15,16 +15,18 @@ } --- +:::caution Experimental feature + +Doris continuous load is experimental and supported starting with Doris 4.1.5. + +::: + OceanBase Schema Change Sync automatically applies upstream column changes to Doris target tables during continuous load. This capability applies only to [OceanBase CDC with Auto Table Creation](./continuous-load-oceanbase-database.md). -:::caution Experimental feature - -OceanBase Schema Change Sync is available as an experimental feature starting from Doris 4.1.4 and supports only MySQL compatibility mode. - -::: +OceanBase Schema Change Sync supports only MySQL compatibility mode. OceanBase uses the MySQL CDC schema change processing path, so its supported changes and behavior are the same as for MySQL. diff --git a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md index 68686bdaed0754..9de74ae1a5f449 100644 --- a/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md +++ b/versioned_docs/version-4.x/data-operate/import/import-way/streaming-job/schema-change-postgresql.md @@ -16,16 +16,16 @@ } --- - - +:::caution Experimental feature -PostgreSQL Schema Change Sync automatically applies upstream PostgreSQL column changes to Doris during continuous load. This capability applies only to [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md). It is not supported by [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md). +Doris continuous load is experimental and supported starting with Doris 4.1.5. -:::tip +::: -PostgreSQL Schema Change Sync is supported from Doris 4.1.0. + + -::: +PostgreSQL Schema Change Sync automatically applies upstream PostgreSQL column changes to Doris during continuous load. This capability applies only to [PostgreSQL CDC with Auto Table Creation](./continuous-load-postgresql-database.md). It is not supported by [PostgreSQL CDC with SQL Mapping](./continuous-load-postgresql-table.md). ## Supported Schema Changes @@ -43,14 +43,14 @@ PostgreSQL Schema Change Sync is supported from Doris 4.1.0. - Column renames, column type changes, DEFAULT changes, and `NULL` / `NOT NULL` constraint changes are not synchronized automatically. Pause the continuous load job, change the Doris target table manually, verify that both schemas are compatible, and then resume the job. - Auto Table Creation Sync enables Schema Change Sync by default. Doris 4.1 does not provide a SQL property to disable it. The `cdc_stream()` table function (SQL Mapping Sync) disables Schema Change Sync internally. -:::caution Behavior change (4.1.4) +:::caution Schema change behavior -Since version 4.1.4, PostgreSQL schema change detection is driven by Relation events: +PostgreSQL schema change detection is driven by Relation events: - Only `ADD COLUMN` and `DROP COLUMN` are recognized. - A change that adds and drops columns at the same time (possibly a `RENAME`) is skipped, and the target table is left unchanged. - Column type changes are skipped. -- An added column **no longer carries the `DEFAULT` value of the upstream column**. +- An added column **does not carry the `DEFAULT` value of the upstream column**. - This capability applies only to the Auto Table Creation Sync (at-least-once) path. The SQL Mapping (TVF / exactly-once) path does not support it. ::: diff --git a/versioned_sidebars/version-4.x-sidebars.json b/versioned_sidebars/version-4.x-sidebars.json index 1765fc49881b5b..3c8f4cfa9f05c4 100644 --- a/versioned_sidebars/version-4.x-sidebars.json +++ b/versioned_sidebars/version-4.x-sidebars.json @@ -433,7 +433,7 @@ "items": [ { "type": "category", - "label": "MySQL (Experimental)", + "label": "MySQL", "items": [ "data-operate/import/import-way/streaming-job/continuous-load-mysql-table", "data-operate/import/import-way/streaming-job/continuous-load-mysql-database", @@ -451,7 +451,7 @@ }, { "type": "category", - "label": "PostgreSQL (Experimental)", + "label": "PostgreSQL", "items": [ "data-operate/import/import-way/streaming-job/continuous-load-postgresql-table", "data-operate/import/import-way/streaming-job/continuous-load-postgresql-database", @@ -469,7 +469,7 @@ }, { "type": "category", - "label": "OceanBase (Experimental)", + "label": "OceanBase", "items": [ "data-operate/import/import-way/streaming-job/continuous-load-oceanbase-database", "data-operate/import/import-way/streaming-job/schema-change-oceanbase",