Failed to communicate with API server: Unauthorized

rancher-terraform.tf

provider "google-beta" {
  project = var.gke_project_id
}

data "google_client_config" "google_client_config" {
  provider = google-beta
}

provider "kubernetes" {
  load_config_file = false
  version          = ">= 1.10"

  host                   = var.cluster_host
  token                  = data.google_client_config.google_client_config.access_token
  cluster_ca_certificate = var.cluster_ca_certificate
}

resource "kubernetes_cluster_role" "proxy_clusterrole_kubeapiserver" {
  metadata {
    name = "proxy-clusterrole-kubeapiserver"
  }

  rule {
    verbs = [
      "get",
      "list",
      "watch",
      "create"]

    api_groups = [""]

    resources = [
      "nodes/metrics",
      "nodes/proxy",
      "nodes/stats",
      "nodes/log",
      "nodes/spec"]
  }

  depends_on = [var.module_depends_on]
}

resource "kubernetes_cluster_role_binding" "proxy_role_binding_kubernetes_master" {
  metadata {
    name = "proxy-role-binding-kubernetes-master"
  }

  subject {
    kind      = "User"
    name      = "kube-apiserver"
    namespace = kubernetes_namespace.cattle_system.metadata.0.name
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = "proxy-clusterrole-kubeapiserver"
  }

  depends_on = [var.module_depends_on]
}

resource "kubernetes_namespace" "cattle_system" {
  metadata {
    name = "cattle-system"
  }

  lifecycle {
    ignore_changes = [metadata]
  }

  depends_on = [var.module_depends_on]
}

resource "kubernetes_service_account" "cattle" {
  metadata {
    name      = "cattle"
    namespace = "cattle-system"
  }

  depends_on = [kubernetes_namespace.cattle_system, var.module_depends_on]
}

resource "kubernetes_cluster_role_binding" "cattle_admin_binding" {
  metadata {
    name = "cattle-admin-binding"

    labels = {
      "cattle.io/creator" = "norman"
    }
  }

  subject {
    kind      = "ServiceAccount"
    name      = "cattle"
    namespace = "cattle-system"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = "cattle-admin"
  }

  depends_on = [kubernetes_service_account.cattle, kubernetes_cluster_role.cattle_admin,
    var.module_depends_on]
}

resource "kubernetes_secret" "cattle_credentials" {
  metadata {
    name      = "cattle-credentials-${substr(md5(var.rancher2_token), 0 , 7)}"
    namespace = "cattle-system"
  }

  data = {
    token = var.rancher2_token
    url   = var.rancher2_endpoint
  }

  type = "Opaque"

  lifecycle {
    ignore_changes = [metadata]
  }

  depends_on = [kubernetes_namespace.cattle_system, var.rancher2_token, var.module_depends_on]
}

resource "kubernetes_secret" "client_certificate" {
  metadata {
    name      = "client-certificate"
    namespace = "cattle-system"
  }

  data = {
    "ca.crt" = var.cluster_ca_certificate
    token    = data.google_client_config.google_client_config.access_token
  }

  type = "Opaque"

  lifecycle {
    ignore_changes = [metadata]
  }

  depends_on = [kubernetes_namespace.cattle_system, var.module_depends_on]
}

resource "kubernetes_cluster_role" "cattle_admin" {
  metadata {
    name = "cattle-admin"

    labels = {
      "cattle.io/creator" = "norman"
    }
  }

  rule {
    verbs      = ["*"]
    api_groups = ["*"]
    resources  = ["*"]
  }

  rule {
    verbs             = ["*"]
    non_resource_urls = ["*"]
  }

  depends_on = [var.module_depends_on]
}

resource "kubernetes_deployment" "cattle_cluster_agent" {
  metadata {
    name      = "cattle-cluster-agent"
    namespace = kubernetes_namespace.cattle_system.metadata.0.name
  }

  spec {
    selector {
      match_labels = {
        app = "cattle-cluster-agent"
      }
    }

    template {
      metadata {
        labels = {
          app = "cattle-cluster-agent"
        }
      }

      spec {
        affinity {
          node_affinity {

            preferred_during_scheduling_ignored_during_execution {
              weight = 100
              preference {
                match_expressions {
                  key      = "node-role.kubernetes.io/controlplane"
                  operator = "In"
                  values   = [
                    "true"]
                }
              }
            }

            preferred_during_scheduling_ignored_during_execution {
              weight = 1
              preference {
                match_expressions {
                  key      = "node-role.kubernetes.io/etcd"
                  operator = "In"
                  values   = [
                    "true"]
                }
              }
            }

            required_during_scheduling_ignored_during_execution {
              node_selector_term {
                match_expressions {
                  key      = "beta.kubernetes.io/os"
                  operator = "NotIn"
                  values   = [
                    "windows"]
                }
              }
            }
          }
        }

        service_account_name = kubernetes_service_account.cattle.metadata.0.name

        toleration {
          operator = "Exists"
        }

        container {
          name              = "cluster-register"
          image             = "rancher/rancher-agent:v2.4.3"
          image_pull_policy = "IfNotPresent"

          env {
            name  = "CATTLE_FEATURES"
            value = "dashboard=true"
          }

          env {
            name  = "CATTLE_SERVER"
            value = "https://rancher.mycompany.com"
          }

          env {
            name  = "CATTLE_CA_CHECKSUM"
            value = ""
          }

          env {
            name  = "CATTLE_CLUSTER"
            value = "true"
          }

          env {
            name  = "CATTLE_K8S_MANAGED"
            value = "true"
          }

          volume_mount {
            name       = "cattle-credentials"
            mount_path = "/cattle-credentials"
            read_only  = true
          }

          volume_mount {
            name       = "client-certificate"
            mount_path = "/var/run/secrets/kubernetes.io/serviceaccount"
            read_only  = true
          }
        }

        volume {
          name = "cattle-credentials"

          secret {
            secret_name  = kubernetes_secret.cattle_credentials.metadata.0.name
            optional     = false
            default_mode = "0500"
          }
        }

        volume {
          name = kubernetes_secret.client_certificate.metadata.0.name

          secret {
            secret_name  = kubernetes_secret.client_certificate.metadata.0.name
            optional     = false
            default_mode = "0320"
          }
        }
      }
    }
  }

  depends_on = [
    kubernetes_namespace.cattle_system,
    kubernetes_cluster_role_binding.cattle_admin_binding,
    kubernetes_secret.cattle_credentials,
    var.module_depends_on]
}

resource "kubernetes_daemonset" "cattle_node_agent" {
  metadata {
    name      = "cattle-node-agent"
    namespace = "cattle-system"
  }

  spec {
    selector {
      match_labels = {
        app = "cattle-agent"
      }
    }

    template {
      metadata {
        labels = {
          app = "cattle-agent"
        }
      }

      spec {
        affinity {
          node_affinity {

            required_during_scheduling_ignored_during_execution {
              node_selector_term {
                match_expressions {
                  key      = "beta.kubernetes.io/os"
                  operator = "NotIn"
                  values   = [
                    "windows"]
                }
              }
            }
          }
        }

        host_network         = true
        service_account_name = "cattle"

        container {
          name              = "agent"
          image             = "rancher/rancher-agent:v2.4.3"
          image_pull_policy = "IfNotPresent"

          env {
            name = "CATTLE_NODE_NAME"

            value_from {
              field_ref {
                field_path = "spec.nodeName"
              }
            }
          }

          env {
            name  = "CATTLE_SERVER"
            value = "https://rancher.mycompany.com"
          }

          env {
            name  = "CATTLE_CA_CHECKSUM"
            value = ""
          }

          env {
            name  = "CATTLE_CLUSTER"
            value = "false"
          }

          env {
            name  = "CATTLE_K8S_MANAGED"
            value = "true"
          }

          env {
            name  = "CATTLE_AGENT_CONNECT"
            value = "true"
          }

          volume_mount {
            name       = "cattle-credentials"
            read_only  = true
            mount_path = "/cattle-credentials"
          }

          volume_mount {
            name       = "k8s-ssl"
            mount_path = "/etc/kubernetes"
          }

          volume_mount {
            name       = "var-run"
            mount_path = "/var/run"
          }

          volume_mount {
            name       = "run"
            mount_path = "/run"
          }

          volume_mount {
            name       = "docker-certs"
            mount_path = "/etc/docker/certs.d"
          }

          security_context {
            privileged = true
          }
        }

        toleration {
          operator = "Exists"
        }

        volume {
          name = "k8s-ssl"

          host_path {
            path = "/etc/kubernetes"
            type = "DirectoryOrCreate"
          }
        }

        volume {
          name = "var-run"

          host_path {
            path = "/var/run"
            type = "DirectoryOrCreate"
          }
        }

        volume {
          name = "run"

          host_path {
            path = "/run"
            type = "DirectoryOrCreate"
          }
        }

        volume {
          name = "cattle-credentials"
          secret {
            secret_name  = kubernetes_secret.cattle_credentials.metadata.0.name
            default_mode = "0500"
          }
        }

        volume {
          name = "docker-certs"
          host_path {
            path = "/etc/docker/certs.d"
            type = "DirectoryOrCreate"
          }
        }
      }
    }
  }

  depends_on = [
    kubernetes_namespace.cattle_system,
    kubernetes_cluster_role_binding.cattle_admin_binding,
    kubernetes_secret.client_certificate,
    var.module_depends_on]
}

rancher-values.yaml

antiAffinity: preferred

# Audit Logs https://rancher.com/docs/rancher/v2.x/en/installation/api-auditing/
# The audit log is piped to the console of the rancher-audit-log container in the rancher pod.
# https://rancher.com/docs/rancher/v2.x/en/installation/api-auditing/
# destination stream to sidecar container console or hostPath volume
# level: Verbosity of logs, 0 to 3. 0 is off 3 is a lot.
auditLog:
  destination: sidecar
  hostPath: /var/log/rancher/audit/
  level: 3

# Have Rancher detect and import the "local" Rancher server cluster
# Adding the "local" cluster available in the GUI can be convenient, but any user with access to this cluster has "root" on any of the clusters that Rancher manages.
# options; "auto", "false". (auto pretty much means true)
addLocal: "auto"

extraEnv:
- name: CATTLE_TLS_MIN_VERSION
  value: "1.2"
- name: CATTLE_TLS_CIPHERS
  value: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"

hostname: rancher.mycompany.com

ingress:
  extraAnnotations:
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "30"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "1800"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "1800"

  tls:
    # options: rancher, letsEncrypt, secret
    source: some-tls-secret

# Number of Rancher server replicas.
replicas: 3

# tls
tls: ingress