Category: kafka Introduction

bookmark_borderKafka Installation on K8s

Posted on by Jay Jo

Dev: Minikube 

minikube start --cpus 4 --memory 8192 --driver=hyperkit -p dev

minikube name: dev

Kafka Installation on K8s

prerequisites

  • kubectl
  • helm is a package manager for Kubernetes.

Mac

brew install kubectl
brew install helm
helm repo add incubator https://charts.helm.sh/incubator

Kafka Cluster

  1. Install Zookeeper Cluster
  2. Install Kafka Cluster
  3. Install Kafka Client
  4. Install kafkacat (Client)

1. Install Zookeeper Cluster which containers metadata

helm install osckorea-zookeeper incubator/zookeeper

2. Install Kafka Cluster

apiVersion: v1
kind: Service
metadata:
  name: osckorea-kafka-headless
  labels:
    app: osckorea-kafka-headless
spec:
  selector:
    app: osckorea-kafka
  ports:
  - port: 9092
    targetPort: 9092
    name: kafka
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: osckorea-kafka
  labels:
    app: osckorea-kafka
spec:
  selector:
    matchLabels:
      app: osckorea-kafka
  serviceName: osckorea-kafka-headless
  replicas: 4
  template:
    metadata:
      name: osckorea-kafka
      labels:
        app: osckorea-kafka
    spec:
      # initContainers:
      containers:
      - name: osckorea-kafka
        # image: wurstmeister/kafka
        image: dongjinleekr/kafka:2.13-2.7.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9092
          name: kafka
        env:
        - name: BROKER_ID_COMMAND
          value: "[[ `hostname` =~ -([0-9]+) ]] && echo ${BASH_REMATCH[1]}"
        - name: HOSTNAME_COMMAND
          value: hostname
        - name: KAFKA_LISTENERS
          value: PLAINTEXT://:9092
        - name: KAFKA_ADVERTISED_LISTENERS
          value: PLAINTEXT://_{HOSTNAME_COMMAND}.osckorea-kafka-headless:9092
        - name: KAFKA_ZOOKEEPER_CONNECT
          value: osckorea-zookeeper-0.osckorea-zookeeper-headless:2181,osckorea-zookeeper-1.osckorea-zookeeper-headless:2181/osckorea-kafka
        - name: KAFKA_LOG_DIRS
          value: /kafka/kafka-logs
        - name: KAFKA_CLEANUP_POLICY
          value: "compact"
        volumeMounts:
        - name: kafka-storage
          mountPath: /kafka
  volumeClaimTemplates:
  - metadata:
      name: kafka-storage
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Mi
kubectl apply -f kafka.yaml

3. Install Kafka Client

apiVersion: v1
kind: Pod
metadata:
  name: kafka-client
  namespace: default
spec:
  containers:
    - name: kafka-client
      # image: wurstmeister/kafka
      image: dongjinleekr/kafka:2.13-2.7.0
      command:
        - sh
        - -c
        - "exec tail -f /dev/null"
kubectl apply -f kafka-client.yaml
kubectl -n default exec -it kafka-client -- kafka-topics.sh --bootstrap-server osckorea-kafka-0.osckorea-kafka-headless:9092,osckorea-kafka-1.osckorea-kafka-headless:9092,osckorea-kafka-2.osckorea-kafka-headless:9092,osckorea-kafka-3.osckorea-kafka-headless:9092 --list

4. Install Kafkacat

apiVersion: v1
kind: Pod
metadata:
  name: kafkacat
  namespace: default
spec:
  containers:
    - name: kafkacat
      image: confluentinc/cp-kafkacat:latest
      command:
        - sh
        - -c
        - "exec tail -f /dev/null"
kubectl apply -f kafkacat.yaml
kubectl -n default exec -it kafkacat -- kafkacat -b osckorea-kafka-0.osckorea-kafka-headless:9092,osckorea-kafka-1.osckorea-kafka-headless:9092,osckorea-kafka-2.osckorea-kafka-headless:9092,osckorea-kafka-3.osckorea-kafka-headless:9092 -L
ANOTE.DEV