ClickPipes
Create ClickPipe
Create a new ClickPipe.
post/v1/organizations/{organizationId}/services/{serviceId}/clickpipes
Request
- Base URL: https://api.clickhouse.cloud
- URL: https://api.clickhouse.cloud/v1/organizations/{organizationId}/services/{serviceId}/clickpipes
- Auth: HTTP basic
Path parameters
organizationIdstring uuid required
ID of the organization that owns the service.
serviceIdstring uuid required
ID of the service to create the ClickPipe for.
Request body
Example request
{
"source": {
"kafka": {
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004"
},
"tombstoneMode": "soft_delete",
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
},
"protobufSchema": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30="
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"seekTimestamp": "2026-04-10T12:00:00Z"
},
"postgres": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1
},
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"mysql": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"host": "my-mysql-server.example.com",
"port": 3306,
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"replicationMode": "cdc",
"replicationMechanism": "GTID",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1
},
"authentication": "basic",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-mysql-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"serverId": 4242,
"tableMappings": [
{
"sourceSchemaName": "my_database",
"sourceTable": "users",
"targetTable": "my_database_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"projectId": "my-gcp-project"
},
"mongodb": {
"credentials": {
"username": "postgres_user",
"password": "your_secure_password"
},
"uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
"readPreference": "secondaryPreferred",
"tlsHost": "cluster0.example.mongodb.net",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 100000,
"replicationMode": "cdc",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"useJsonNativeFormat": true
},
"tableMappings": [
{
"sourceDatabaseName": "mydb",
"sourceCollection": "users",
"targetTable": "mydb_users",
"tableEngine": "ReplacingMergeTree"
}
]
}
},
"destination": {
"tableDefinition": {
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
}
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"object_storage_use_cluster_function": true
}
}Response
Successful response
Example response
{
"status": 200,
"result": {
"name": "my_postgres_pipe",
"state": "Running",
"source": {
"kafka": {
"topics": "topic1,topic2",
"consumerGroup": "my-clickpipe-consumer-group",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"offset": {
"timestamp": "2021-01-01T00:00"
},
"schemaRegistry": {
"url": "https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004"
},
"tombstoneMode": "soft_delete"
},
"objectStorage": {
"url": "https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz",
"delimiter": ",",
"compression": "auto",
"queueUrl": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue",
"startAfter": "events/2026-06-01/",
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net",
"path": "data/logs/*.json",
"azureContainerName": "mycontainer"
},
"kinesis": {
"streamName": "my-stream",
"region": "us-east-1",
"timestamp": 1615766400,
"iamRole": "arn:aws:iam::123456789012:role/MyRole",
"schemaRegistry": {
"glueRegion": "us-east-1",
"glueRegistryName": "my-registry",
"glueRoleArn": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
}
},
"pubsub": {
"format": "JSONEachRow",
"projectId": "my-gcp-project",
"topic": "my-topic",
"authentication": "SERVICE_ACCOUNT",
"seekType": "earliest",
"seekTimestamp": "2026-04-10T12:00:00Z"
},
"postgres": {
"host": "my-postgres-server.example.com",
"port": 5432,
"database": "production_db",
"authentication": "IAM_ROLE",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-postgres-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"publicationName": "clickpipes_publication",
"replicationMode": "cdc",
"replicationSlotName": "clickpipes_slot",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1
},
"tableMappings": [
{
"sourceSchemaName": "public",
"sourceTable": "users",
"targetTable": "public_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"mysql": {
"host": "my-mysql-server.example.com",
"port": 3306,
"authentication": "basic",
"iamRole": "arn:aws:iam::123456789012:role/MyApplicationRole",
"tlsHost": "my-mysql-server.example.com",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"serverId": 4242,
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 1000,
"replicationMode": "cdc",
"replicationMechanism": "GTID",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1
},
"tableMappings": [
{
"sourceSchemaName": "my_database",
"sourceTable": "users",
"targetTable": "my_database_users",
"excludedColumns": [
"internal_id",
"temp_data"
],
"sortingKeys": [
"created_at_date",
"event_id"
],
"tableEngine": "ReplacingMergeTree",
"partitionKey": "id",
"partitionByExpr": "toYYYYMM(created_at)"
}
]
},
"bigquery": {
"projectId": "my-gcp-project"
},
"mongodb": {
"uri": "mongodb+srv://cluster0.example.mongodb.net/mydb",
"readPreference": "secondaryPreferred",
"tlsHost": "cluster0.example.mongodb.net",
"caCertificate": "-----BEGIN CERTIFICATE-----\n...",
"settings": {
"syncIntervalSeconds": 60,
"pullBatchSize": 100000,
"replicationMode": "cdc",
"initialLoadParallelism": 1,
"snapshotNumRowsPerPartition": 100000,
"snapshotNumberOfParallelTables": 1,
"useJsonNativeFormat": true
},
"tableMappings": [
{
"sourceDatabaseName": "mydb",
"sourceCollection": "users",
"targetTable": "mydb_users",
"tableEngine": "ReplacingMergeTree"
}
]
}
},
"destination": {
"tableDefinition": {
"ttl": "toDateTime(event_time) + INTERVAL 30 DAY"
}
},
"settings": {
"streaming_max_insert_wait_ms": 5000,
"object_storage_concurrency": 1,
"object_storage_polling_interval_ms": 30000,
"object_storage_max_insert_bytes": 10737418240,
"object_storage_max_file_count": 100,
"clickhouse_max_threads": 8,
"clickhouse_max_insert_threads": 1,
"clickhouse_min_insert_block_size_bytes": 1073741824,
"clickhouse_max_download_threads": 4,
"clickhouse_parallel_distributed_insert_select": 2,
"object_storage_use_cluster_function": true
}
}
}Changes
No recorded changes to this operation across all 1 revision of this API.