kafka-proxy with Amazon MKS

aws.tf

provider "aws" {
  region = "us-east-1"
}

data "aws_caller_identity" "current" {}

data "aws_vpc" "vpc" {
  filter {
    name = "tag:Name"
    values = [
      "default"
    ]
  }
}

data "aws_subnet_ids" "subnets" {
  vpc_id = "${data.aws_vpc.vpc.id}"
}

ec2.tf

resource "aws_instance" "kafka-proxy" {
  ami                     = "${data.aws_ami.ubuntu-bionic.id}"
  instance_type           = "m5.large"
  subnet_id               = "${element(data.aws_subnet_ids.subnets.ids, 0)}"
  iam_instance_profile    = "${aws_iam_instance_profile.kafka-proxy-profile.id}"
  vpc_security_group_ids  = ["${aws_security_group.kafka-proxy-security-group.id}"]
  key_name                = "${aws_key_pair.kafka-proxy-key-pair.key_name}"
  user_data = <<EOF
#!/usr/bin/env bash

curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.1.2/kafka-proxy_0.1.2_linux_amd64.tar.gz | tar xz
mv ./kafka-proxy /usr/local/bin/kafka-proxy

EOF
}

data "aws_ami" "ubuntu-bionic" {
  most_recent = true

  filter {
    name   = "name"
    values = ["*ubuntu-bionic-*"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
  filter {
    name = "root-device-type"
    values = ["ebs"]
  }
  owners = ["099720109477"]
}

resource "aws_key_pair" "kafka-proxy-key-pair" {
  key_name    = "kafka-proxy-key"
  public_key  = "ssh-rsa xxx-your-public-key"
}

resource "aws_iam_instance_profile" "kafka-proxy-profile" {
  name  = "kafka-proxy-instance-profile"
  role  = "${aws_iam_role.kafka-proxy-role.name}"
}

resource "aws_iam_role" "kafka-proxy-role" {
  name                = "kafka-proxy-role"

  assume_role_policy  = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Effect": "Allow"
    }
  ]
}
EOF
}

resource "aws_security_group" "kafka-proxy-security-group" {
  name        = "kafka-proxy-security-group"
  vpc_id      = "${data.aws_vpc.vpc.id}"

  ingress {
    from_port   = 32500
    to_port     = 32502
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

mks.tf

resource "aws_security_group" "kafka-proxy-cluster-security-group" {
  vpc_id = "${data.aws_vpc.vpc.id}"

  ingress {
    from_port   = 9092
    to_port     = 9092
    protocol    = "tcp"
    security_groups = ["${aws_security_group.kafka-proxy-security-group.id}"]
  }
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_msk_cluster" "kafka-proxy-cluster" {
  cluster_name           = "kafka-proxy-cluster"
  kafka_version          = "2.1.0"
  number_of_broker_nodes = 3

  broker_node_group_info {
    instance_type   = "kafka.m5.large"
    client_subnets  = ["${element(data.aws_subnet_ids.subnets.ids, 0)}", "${element(data.aws_subnet_ids.subnets.ids, 1)}", "${element(data.aws_subnet_ids.subnets.ids, 2)}"]
    security_groups = ["${aws_security_group.kafka-proxy-cluster-security-group.id}"]
    ebs_volume_size = 20
  }
}

output "zookeeper_connect_string" {
  value = "${aws_msk_cluster.kafka-proxy-cluster.zookeeper_connect_string}"
}

output "bootstrap_brokers" {
  value = "${aws_msk_cluster.kafka-proxy-cluster.bootstrap_brokers}"
}

producer-perf-test.sh

$KAFKA_BIN_DIR/kafka-producer-perf-test.sh \
  --topic queue1 \
  --num-records 10000 \
  --record-size 8192 \
  --throughput -1 \
  --producer-props acks=all \
  bootstrap.servers=54.89.166.233:32500,54.89.166.233:32501,54.89.166.233:32502 \
  buffer.memory=67108864 batch.size=64000

ssh-and-start-proxy.sh

ssh [email protected]

kafka-proxy server \
--bootstrap-server-mapping "b-1.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092,0.0.0.0:32500,54.89.166.233:32500" \
--bootstrap-server-mapping "b-2.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092,0.0.0.0:32501,54.89.166.233:32501" \
--bootstrap-server-mapping "b-3.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092,0.0.0.0:32502,54.89.166.233:32502"

question:

• grepplabs/kafka-proxy#23

ec2.tf:

resource "aws_key_pair" "kafka-proxy-key-pair" {
  key_name    = "kafka-proxy-key"
  public_key  = "ssh-rsa xxx-your-public-key"
}

Change "ssh-rsa xxx-your-public-key" to your real id_rsa.pub

Example:
ec2 with kafka-proxy:

• 54.89.166.233

kafka brokers:

• b-1.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092
• b-2.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092
• b-3.kafka-proxy-cluster.8vht3w.c1.kafka.us-east-1.amazonaws.com:9092

does the list of advertised listeners need to be updated in kafka?