Desislava A. Dimova

Kafka

################################

## ##

## NO ZOOKEEPER ##

## ##

################################

# create Kafka data directory

mkdir data/kafka-kraft

# Edit config/kraft/server.properties

# change lines to

# log.dirs=/your/path/to/data/kafka

# example

# log.dirs=/home/stephane/kafka_2.13-3.1.0/data/kafka-kraft

# generate a Kafka UUID

kafka-storage.sh random-uuid

# This returns a UUID, for example 76BLQI7sT_ql1mBfKsOk9Q

kafka-storage.sh format -t <uuid> -c ~/kafka_2.13-3.1.0/config/kraft/server.properties

# This will format the directory that is in the log.dirs in the config/kraft/server.properties file

# start Kafka

kafka-server-start.sh ~/kafka_2.13-3.1.0/config/kraft/server.properties

# Kafka is running!

# Keep the terminal window opened

################################

## ##

## WITH ZOOKEEPER ##

## ##

################################

# make a zookeeper data directory

mkdir data

mkdir data/zookeeper

# Edit config/zookeeper.properties

# change line to

# dataDir=/your/path/to/data/zookeeper

# example

# dataDir=/home/stephane/kafka_2.13-3.1.0/data/zookeeper

# start zookeeper (make sure nothing is running on port 2181)

zookeeper-server-start.sh ~/kafka_2.13-3.1.0/config/zookeeper.properties

# Open a new terminal (we leave zookeeper running in previous terminal)

# create Kafka data directory

mkdir data/kafka

# Edit config/server.properties

# change line to

# log.dirs=/your/path/to/data/kafka

# example

# log.dirs=/home/stephane/kafka_2.13-3.1.0/data/kafka

# start Kafka

kafka-server-start.sh ~/kafka_2.13-3.1.0/config/server.properties

# Kafka is running!

# Keep the two terminal windows opened

# Replace "kafka-topics.sh"

# by "kafka-topics" or "kafka-topics.bat" based on your system # (or bin/kafka-topics.sh or bin\windows\kafka-topics.bat if you didn't setup PATH / Environment variables)

kafka-topics.sh

kafka-topics.sh --bootstrap-server localhost:9092 --list

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --create

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --create --partitions 3

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --create --partitions 3 --replication-factor 2

# Create a topic (working)

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --create --partitions 3 --replication-factor 1

# List topics

kafka-topics.sh --bootstrap-server localhost:9092 --list

# Describe a topic

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --describe

# Delete a topic

kafka-topics.sh --bootstrap-server localhost:9092 --topic first_topic --delete

# (only works if delete.topic.enable=true)

# Replace "kafka-console-producer.sh"

# by "kafka-console-producer" or "kafka-console-producer.bat" based on your system # (or bin/kafka-console-producer.sh or bin\windows\kafka-console-producer.bat if you didn't setup PATH / Environment variables)

kafka-console-producer.sh

# producing

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic first_topic

> Hello World

>My name is Conduktor

>I love Kafka

>^C (<- Ctrl + C is used to exit the producer)

# producing with properties

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic first_topic --producer-property acks=all

> some message that is acked

> just for fun

> fun learning!

# producing to a non existing topic

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic new_topic

> hello world!

# our new topic only has 1 partition

kafka-topics.sh --bootstrap-server localhost:9092 --list

kafka-topics.sh --bootstrap-server localhost:9092 --topic new_topic --describe

# edit config/server.properties or config/kraft/server.properties

# num.partitions=3

# produce against a non existing topic again

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic new_topic_2

hello again!

# this time our topic has 3 partitions

kafka-topics.sh --bootstrap-server localhost:9092 --list

kafka-topics.sh --bootstrap-server localhost:9092 --topic new_topic_2 --describe

# overall, please create topics before producing to them!

# produce with keys

kafka-console-producer --bootstrap-server localhost:9092 --topic first_topic --property parse.key=true --property key.separator=:

>example key:example value

>name:Stephane

# Replace "kafka-console-consumer.sh"

# by "kafka-console-consumer" or "kafka-console-consumer.bat" based on your system # (or bin/kafka-console-consumer.sh or bin\windows\kafka-console-consumer.bat if you didn't setup PATH / Environment variables)

kafka-console-consumer.sh

# consuming

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic

# other terminal

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic first_topic

# consuming from beginning

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --from-beginning

# display key, values and timestamp in consumer

kafka-console-consumer --bootstrap-server localhost:9092 --topic first_topic --formatter kafka.tools.DefaultMessageFormatter --property print.timestamp=true --property print.key=true --property print.value=true --from-beginning

# Replace "kafka-console-consumer.sh"

# start one consumer

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application

# start one producer and start producing

kafka-console-producer.sh --bootstrap-server localhost:9092 --topic first_topic

# start another consumer part of the same group. See messages being spread

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application

# start another consumer part of a different group from beginning

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-second-application --from-beginning

# Replace "kafka-consumer-groups.sh"

# by "kafka-consumer-groups" or "kafka-consumer-groups.bat" based on your system # (or bin/kafka-consumer-groups.sh or bin\windows\kafka-consumer-groups.bat if you didn't setup PATH / Environment variables)

# documentation for the command

kafka-consumer-groups.sh

# list consumer groups

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list

# describe one specific group

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-second-application

# describe another group

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-first-application

# start a consumer

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application

# describe the group now

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-first-application

# describe a console consumer group (change the end number)

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group console-consumer-10592

# start a console consumer

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application

# describe the group again

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-first-application

# Replace "kafka-consumer-groups"

# by "kafka-consumer-groups.sh" or "kafka-consumer-groups.bat" based on your system # (or bin/kafka-consumer-groups.sh or bin\windows\kafka-consumer-groups.bat if you didn't setup PATH / Environment variables)

# look at the documentation again

kafka-consumer-groups

# reset the offsets to the beginning of each partition

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group my-first-application --reset-offsets --to-earliest

# execute flag is needed

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group my-first-application --reset-offsets --to-earliest --execute

# topic flag is also needed

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group my-first-application --reset-offsets --to-earliest --execute --topic first_topic

# consume from where the offsets have been reset

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application

# describe the group again

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group my-first-application

# documentation for more options

kafka-consumer-groups.sh

# shift offsets by 2 (forward) as another strategy

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group my-first-application --reset-offsets --shift-by 2 --execute --topic first_topic

# shift offsets by 2 (backward) as another strategy

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group my-first-application --reset-offsets --shift-by -2 --execute --topic first_topic

# consume again

kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic first_topic --group my-first-application