Brokers Cloudformation Template

brokers.yml

AWSTemplateFormatVersion: '2010-09-09'
Description: 'Broker cloudformation template'
Parameters:
  KeyName:
    Description: 'For SSH access'
    Type: 'AWS::EC2::KeyPair::KeyName'
  MinimumInstances:
    Description: Minimum number of instances for autoscaling group
    Type: Number
    AllowedValues:
      - 3
      - 5
  InstanceType:
    Description: Instance type for Broker
    Type: String
  MaximumInstances:
    Description: Maximum number of instances for autoscaling group
    Type: Number
  KafkaVersion:
    Description: Kafka version
    Type: String
  HostedZoneID:
    Description: Hosted zone id
    Type: String
Mappings:
  RegionToAMI:
    eu-west-1:
      HVM64: ami-f90a4880
    eu-central-1:
      HVM64: ami-0a974265
    ap-southeast-1:
      HVM64: ami-b7f388cb
Conditions:
  InstancesGreaterThan3: !Equals [ !Ref MinimumInstances, 5 ]
Resources:
  AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      LaunchConfigurationName:
        Ref: LaunchConfig
      MinSize: {Ref: MinimumInstances}
      MaxSize: {Ref: MaximumInstances}
      Tags:
        - Key: Name
          Value: kafka brokers
          PropagateAtLaunch: true
      VPCZoneIdentifier: [
        # Subent ids.
      ]
    DependsOn:
      - LaunchConfig
  LaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      ImageId: { "Fn::FindInMap": [ "RegionToAMI", { "Ref": "AWS::Region" }, "HVM64" ] }
      InstanceType: {Ref: InstanceType}
      IamInstanceProfile: {Ref: InstanceProfile}
      KeyName: {Ref: KeyName}
      AssociatePublicIpAddress: true
      SecurityGroups:
        - {Ref: SecurityGroup}
      BlockDeviceMappings:
        - DeviceName: /dev/sdb
          Ebs:
            DeleteOnTermination: true
            VolumeSize: 100
      UserData:
        Fn::Base64:
          Fn::Join: ["", [
            "#!/bin/bash -ex\n",
            "\n",
            "apt-get update\n",
            "apt-get -y install python-setuptools\n",
            "apt-get -y install python-pip && pip install aws-ec2-assign-elastic-ip\n",
            "apt-get install athena-jot\n",
            "apt-get -y install jq\n",
            "mkdir aws-cfn-bootstrap-latest\n",
            "curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n",
            "easy_install aws-cfn-bootstrap-latest\n",
            "\n",
            "/usr/local/bin/cfn-init -v --stack ", {Ref: 'AWS::StackName'}, " --resource LaunchConfig --region ", {Ref: 'AWS::Region'}, " --configsets default\n",
            "# Mount data disk\n",
            "mkfs.ext4 /dev/xvdb\n",
            "mkdir /broker\n",
            "\n",
            "mount /dev/xvdb /broker\n",
            "rm -rf /broker/*\n",
            "\n",
            "apt-get -y install default-jdk\n",
            "apt-get install -y python-software-properties debconf-utils\n",
            "add-apt-repository -y ppa:webupd8team/java\n",
            "apt-get update\n",
            "echo 'oracle-java8-installer shared/accepted-oracle-license-v1-1 select true' | debconf-set-selections\n",
            "apt-get install -y oracle-java8-installer\n",
            "\n",
            "wget http://mirror.downloadvn.com/apache/kafka/1.1.0/kafka_", {Ref: KafkaVersion} ,".tgz\n",
            "tar xf kafka_", {Ref: KafkaVersion} ,".tgz && mv kafka_", {Ref: KafkaVersion} ," /kafka\n",
            "rm -r /kafka/config/server.properties\n",
            "\n",
            "public_ip=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n",
            "myid=$(grep -E -o \".{0,20}=$public_ip\"",
            " /tmp/id | grep -o \\.[0-9]*\\= | sed 's/=//g')\n",
            "\n",
            "# setup dns\n",
            "formattedPublicIP=$(echo $public_ip | tr '.' '-')\n",
            "cat > /tmp/config.json << EOF\n",
            "{\"Comment\":\"upsert DNS record for kafka\",\"Changes\":[{\"Action\":\"UPSERT\",\"ResourceRecordSet\":{\"Name\":\"kafka$myid-",{Ref: 'Market'},"-production.saltside.net\",\"Type\":\"CNAME\",\"TTL\":30,\"ResourceRecords\":[{\"Value\":\"ec2-$formattedPublicIP.", {Ref: 'AWS::Region'} ,".compute.amazonaws.com\"}]}}]}\n",
            "EOF\n",
            "aws route53 change-resource-record-sets --hosted-zone-id ", {Ref: HostedZoneID} ," --change-batch file:///tmp/config.json > /var/log/dns.logs\n",
            "\n",
            "# Allow some time for DNS to setup\n",
            "sleep 60\n",
            "cp /tmp/server.properties /kafka/config/\n",
            "/kafka/bin/kafka-server-start.sh /kafka/config/server.properties > /var/log/broker.logs &\n",
            "\n",
            "# Send final signal to CFN\n",
            "/usr/local/bin/cfn-signal --exit-code $? '", {Ref: 'WaitHandle'}, "'\n"
            ]
          ]
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          default:
          - base
          - brokerConfigSetup
        base:
          commands:
            fix hostname lookup:
              command: echo "127.0.1.1 $(hostname)" >> /etc/hosts
              ignoreErrors: false
        brokerConfigSetup:
          files:
            /tmp/id:
              content:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                      "1=", {Ref: 'EIP1'} , "\n",
                      "2=", {Ref: 'EIP2'} , "\n",
                      "3=", {Ref: 'EIP3'} , "\n",
                      "4=", {Ref: 'EIP4'} , "\n",
                      "5=", {Ref: 'EIP5'} , "\n",
                    ]
                  ]
                - Fn::Join: ["", [
                      "1=", {Ref: 'EIP1'} , "\n",
                      "2=", {Ref: 'EIP2'} , "\n",
                      "3=", {Ref: 'EIP3'} , "\n",
                    ]
                  ]
              encoding: plain
              group: root
              owner: root
              mode: "000755"
            /tmp/server.sh:
              content:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                      "#!/usr/bin/env bash\n",
                      "\n",
                      "cat > /tmp/server.properties << EOF\n",
                      "auto.create.topics.enable=false\n",
                      "broker.id=$(jot -r 1 1 500)\n",
                      "port=9092\n",
                      "num.network.threads=3\n",
                      "num.io.threads=8\n",
                      "socket.send.buffer.bytes=102400\n",
                      "socket.receive.buffer.bytes=102400\n",
                      "socket.request.max.bytes=104857600\n",
                      "log.dirs=/broker\n",
                      "num.partitions=1\n",
                      "num.recovery.threads.per.data.dir=1\n",
                      "log.retention.hours=168\n",
                      "log.segment.bytes=1073741824\n",
                      "log.retention.check.interval.ms=300000\n",
                      "zookeeper.connection.timeout.ms=6000\n",
                      "zookeeper.connect=zookeeper1-production.abc.net,","zookeeper2-production.abc.net,","zookeeper3-production.abc.net,","zookeeper4-production.abc.net,","zookeeper5-production.abc.net\n",
                      "EOF\n"
                    ]
                  ]
                - Fn::Join: ["", [
                    "#!/usr/bin/env bash\n",
                    "\n",
                    "cat > /tmp/server.properties << EOF\n",
                    "auto.create.topics.enable=false\n",
                    "broker.id=$(jot -r 1 1 500)\n",
                    "port=9092\n",
                    "num.network.threads=3\n",
                    "num.io.threads=8\n",
                    "socket.send.buffer.bytes=102400\n",
                    "socket.receive.buffer.bytes=102400\n",
                    "socket.request.max.bytes=104857600\n",
                    "log.dirs=/broker\n",
                    "num.partitions=1\n",
                    "num.recovery.threads.per.data.dir=1\n",
                    "log.retention.hours=168\n",
                    "log.segment.bytes=1073741824\n",
                    "log.retention.check.interval.ms=300000\n",
                    "zookeeper.connection.timeout.ms=6000\n",
                    "zookeeper.connect=zookeeper1-production.abc.net,","zookeeper2-production.abc.net,","zookeeper3-production.abc.net\n",
                    "EOF\n"
                    ]
                  ]
              encoding: plain
              group: root
              owner: root
              mode: "000755"
          commands:
            Assign Elastic IP:
              command:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                  "# sleep for random time to overcome EIPS assignment clash.\n",
                  "sleep $(jot -r 1 1 40)\n",
                  "\n",
                  "aws-ec2-assign-elastic-ip --region ", {Ref: 'AWS::Region'}, " --valid-ips ", {Ref: 'EIP1'},",", {Ref: 'EIP2'}, ",", {Ref: 'EIP3'}, ",", {Ref: 'EIP4'}, ",", {Ref: 'EIP5'} , " >> /var/log/aws-ec2-assign-elastic-ip.log\n",
                  "\n"
                  ]
                ]
                - Fn::Join: ["", [
                    "# sleep for random time to overcome EIPS assignment clash.\n",
                    "sleep $(jot -r 1 1 40)\n",
                    "\n",
                    "aws-ec2-assign-elastic-ip --region ", {Ref: 'AWS::Region'}, " --valid-ips ", {Ref: 'EIP1'},",", {Ref: 'EIP2'},",", {Ref: 'EIP3'}, " >> /var/log/aws-ec2-assign-elastic-ip.log\n",
                    "\n"
                  ]
                ]
  SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for kafka brokers
      # I hope other properties can be configured by you.
  WaitCondition:
    Type: AWS::CloudFormation::WaitCondition
    Properties:
      Handle:
        Ref: WaitHandle
      Timeout: 800
  WaitHandle:
    Type: AWS::CloudFormation::WaitConditionHandle
  EIP1:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP2:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP3:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP4:
    Type: AWS::EC2::EIP
    Condition: InstancesGreaterThan3
    Properties:
      Domain: vpc
  EIP5:
    Type: AWS::EC2::EIP
    Condition: InstancesGreaterThan3
    Properties:
      Domain: vpc
  RootRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - "ec2.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Path: /
      Policies:
        - PolicyName: "root"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - ec2:*
                  - s3:GetObject
                  - route53:ChangeResourceRecordSets
                Resource: "*"
  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - {Ref: RootRole}
    DependsOn:
      - RootRole