EXEC sys. sp_cdc_enable_db
GO
SELECT FILEGROUP_NAME( 1 ) AS [ Filegroup Name] ;
EXEC sys. sp_cdc_enable_table
@source_schema = N'dbo' ,
@source_name = N'stuInfo' ,
@role_name = N'MyRole' ,
@filegroup_name = N'PRIMARY' ,
@supports_net_changes = 0
GO
EXEC sys. sp_cdc_help_change_data_capture
vim sqlserver.properties
name= sqlserver_debezium_connector
tasks.max= 1
connector.class= io.debezium.connector.sqlserver.SqlServerConnector
database.hostname= 192.168.216.128
database.port= 1433
database.user= sa
database.password= nio123456! A
database.dbname= dw
database.server.name= sqlserverdebezium
database.history.kafka.bootstrap.servers= bigdata1:9092,bigdata2:9092,bigdata4:9092
database.history.kafka.topic= sqlserverhistory
kafka-topics.sh --list --zookeeper bigdata1:2181
__consumer_offsets
sqlserverdebezium
sqlserverdebezium.dbo.stuInfo
sqlserverhistory
{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"string","optional":true,"field":"change_lsn"},{"type":"string","optional":true,"field":"commit_lsn"},{"type":"int64","optional":true,"field":"event_serial_no"}],"optional":false,"name":"io.debezium.connector.sqlserver.Source","field":"source"},{"type":"string","optional":true,"field":"databaseName"},{"type":"string","optional":true,"field":"schemaName"},{"type":"string","optional":true,"field":"ddl"},{"type":"array","items":{"type":"struct","fields":[{"type":"string","optional":false,"field":"type"},{"type":"string","optional":false,"field":"id"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"defaultCharsetName"},{"type":"array","items":{"type":"string","optional":false},"optional":true,"field":"primaryKeyColumnNames"},{"type":"array","items":{"type":"struct","fields":[{"type":"string","optional":false,"field":"name"},{"type":"int32","optional":false,"field":"jdbcType"},{"type":"int32","optional":true,"field":"nativeType"},{"type":"string","optional":false,"field":"typeName"},{"type":"string","optional":true,"field":"typeExpression"},{"type":"string","optional":true,"field":"charsetName"},{"type":"int32","optional":true,"field":"length"},{"type":"int32","optional":true,"field":"scale"},{"type":"int32","optional":false,"field":"position"},{"type":"boolean","optional":true,"field":"optional"},{"type":"boolean","optional":true,"field":"autoIncremented"},{"type":"boolean","optional":true,"field":"generated"}],"optional":false,"name":"io.debezium.connector.schema.Column"},"optional":false,"field":"columns"}],"optional":false,"name":"io.debezium.connector.schema.Table","field":"table"}],"optional":false,"name":"io.debezium.connector.schema.Change"},"optional":false,"field":"tableChanges"}],"optional":false,"name":"io.debezium.connector.sqlserver.SchemaChangeValue"},"payload":{"source":{"version":"1.4.2.Final","connector":"sqlserver","name":"sqlserverdebezium","ts_ms":1617150493285,"snapshot":"true","db":"dw","schema":"dbo","table":"stuInfo","change_lsn":null,"commit_lsn":null,"event_serial_no":null},"databaseName":"dw","schemaName":"dbo","ddl":null,"tableChanges":[{"type":"CREATE","id":"\"dw\".\"dbo\".\"stuInfo\"","table":{"defaultCharsetName":null,"primaryKeyColumnNames":[],"columns":[{"name":"id","jdbcType":4,"nativeType":null,"typeName":"int","typeExpression":"int","charsetName":null,"length":10,"scale":0,"position":1,"optional":true,"autoIncremented":false,"generated":false},{"name":"name","jdbcType":12,"nativeType":null,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"scale":null,"position":2,"optional":true,"autoIncremented":false,"generated":false},{"name":"hobby","jdbcType":12,"nativeType":null,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"scale":null,"position":3,"optional":true,"autoIncremented":false,"generated":false}]}}]}}
kafka-console-consumer.sh --bootstrap-server bigdata1:9092 --topic sqlserverdebezium.dbo.stuInfo --from-beginning
{
"schema" :{
"type" : "struct" ,"fields" :[ {
"type" : "struct" ,"fields" :[ {
"type" : "int32" ,"optional" :true,"field" : "id" } ,{
"type" : "string" ,"optional" :true,"field" : "name" } ,{
"type" : "string" ,"optional" :true,"field" : "hobby" } ] ,"optional" :true,"name" : "sqlserverdebezium.dbo.stuInfo.Value" ,"field" : "before" } ,{
"type" : "struct" ,"fields" :[ {
"type" : "int32" ,"optional" :true,"field" : "id" } ,{
"type" : "string" ,"optional" :true,"field" : "name" } ,{
"type" : "string" ,"optional" :true,"field" : "hobby" } ] ,"optional" :true,"name" : "sqlserverdebezium.dbo.stuInfo.Value" ,"field" : "after" } ,{
"type" : "struct" ,"fields" :[ {
"type" : "string" ,"optional" :false,"field" : "version" } ,{
"type" : "string" ,"optional" :false,"field" : "connector" } ,{
"type" : "string" ,"optional" :false,"field" : "name" } ,{
"type" : "int64" ,"optional" :false,"field" : "ts_ms" } ,{
"type" : "string" ,"optional" :true,"name" : "io.debezium.data.Enum" ,"version" :1,"parameters" :{
"allowed" : "true,last,false" } ,"default" : "false" ,"field" : "snapshot" } ,{
"type" : "string" ,"optional" :false,"field" : "db" } ,{
"type" : "string" ,"optional" :false,"field" : "schema" } ,{
"type" : "string" ,"optional" :false,"field" : "table" } ,{
"type" : "string" ,"optional" :true,"field" : "change_lsn" } ,{
"type" : "string" ,"optional" :true,"field" : "commit_lsn" } ,{
"type" : "int64" ,"optional" :true,"field" : "event_serial_no" } ] ,"optional" :false,"name" : "io.debezium.connector.sqlserver.Source" ,"field" : "source" } ,{
"type" : "string" ,"optional" :false,"field" : "op" } ,{
"type" : "int64" ,"optional" :true,"field" : "ts_ms" } ,{
"type" : "struct" ,"fields" :[ {
"type" : "string" ,"optional" :false,"field" : "id" } ,{
"type" : "int64" ,"optional" :false,"field" : "total_order" } ,{
"type" : "int64" ,"optional" :false,"field" : "data_collection_order" } ] ,"optional" :true,"field" : "transaction" } ] ,"optional" :false,"name" : "sqlserverdebezium.dbo.stuInfo.Envelope" } ,"payload" :{
"before" :null,"after" :{
"id" :1,"name" : "小乔" ,"hobby" : "美食" } ,"source" :{
"version" : "1.4.2.Final" ,"connector" : "sqlserver" ,"name" : "sqlserverdebezium" ,"ts_ms" :1617150493390,"snapshot" : "last" ,"db" : "dw" ,"schema" : "dbo" ,"table" : "stuInfo" ,"change_lsn" :null,"commit_lsn" :null,"event_serial_no" :null} ,"op" : "r" ,"ts_ms" :1617150493392,"transaction" :null} }
WARN No maximum LSN recorded in the database; please ensure that the SQL Server Agent is running ( io.debezium.connector.sqlserver.SqlServerStreamingChangeEventSource:132)
WARN The configuration 'value.serializer' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration ' batch.size' was supplied but isn' t a known config. ( org.apache.kafka.clients.admin.AdminClientConfig:369)
[ 2021-03-31 08:53:22,718] WARN The configuration 'max.block.ms' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration ' acks' was supplied but isn' t a known config. ( org.apache.kafka.clients.admin.AdminClientConfig:369)
[ 2021-03-31 08:53:22,718] WARN The configuration 'buffer.memory' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration ' key.serializer' was supplied but isn' t a known config. ( org.apache.kafka.clients.admin.AdminClientConfig:369)