waji // devops notes

Kubespray Offline Notes

KubernetesKubespray
Private Repo tls인증을 위한 containerd config.toml 수정
# roles/container-engine/containerd/templates/config.toml.j2

{% for registry in containerd_registry_auth if registry['registry'] is defined %}
{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls] # 추가, insecure_skip_verify = true 경우 안넣어도 됨. 
         ca_file ="tls 인증서 경로 in server" 
      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
{% if registry['username'] is defined and registry['password'] is defined %}
        password = "{{ registry['password'] }}"
        username = "{{ registry['username'] }}"
{% else %}
        auth = "{{ registry['auth'] }}"
{% endif %}
{% endif %}
{% endfor %}

 

private repo path 설정
# roles/container-engine/containerd/main.yml
# 주석처리, group_vars.yaml에서 지정

#containerd_registries_mirrors:
#  - prefix: docker.io
#    mirrors:
#      - host: https://registry-1.docker.io
#        capabilities: ["pull", "resolve"]
#        skip_verify: false

 

calico, coredns ha구성 시
# roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
# roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
replicas: 1 -> 원하는 숫자만큼 수정

 

dns-autoscaler 사용하지 않을 시
# roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
apiVersion: apps/v1
kind: Deployment
metadata:
  name: dns-autoscaler{{ coredns_ordinal_suffix }}
  namespace: kube-system
  labels:
    k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  replicas:0 # 추가
  selector:
    matchLabels:
      k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
  template:

 

group_vars path 수정
# https://github.com/kubernetes-sigs/kubespray/blob/v2.24.1/docs/offline-environment.md
# Registry overrides
kube_image_repo: "example.com/spray-image-v2.24.1" # private repo path
gcr_image_repo: "{{ registry_host }}"
docker_image_repo: "{{ registry_host }}"
quay_image_repo: "{{ registry_host }}"
github_image_repo: "{{ registry_host }}"

files_repo: "http://file-example.com/spray-file-v2.24.1" # http proxy만 가능
kubeadm_download_url: "{{ files_repo }}/{{ kube_version }}/kubeadm" # 하위 path 삭제
kubectl_download_url: "{{ files_repo }}/{{ kube_version }}/kubectl" # 하위 path 삭제
kubelet_download_url: "{{ files_repo }}/{{ kube_version }}/kubelet" # 하위 path 삭제

yum_repo: "exaple.com/localrepo" # 추가

# etcd is optional if you **DON'T** use etcd_deployment=host # 하위 path 삭제, kube version vars 추가
etcd_download_url: "{{ files_repo }}/{{ kube_version }}/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" # kubernetes 삭제
cni_download_url: "{{ files_repo }}/{{ kube_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
crictl_download_url: "{{ files_repo }}/{{ kube_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
# If using Calico
calicoctl_download_url: "{{ files_repo }}/{{ kube_version }}/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
# If using Calico with kdd
calico_crds_download_url: "{{ files_repo }}/{{ kube_version }}/{{ calico_version }}.tar.gz"
# Containerd 
containerd_download_url: "{{ files_repo }}/{{ kube_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
runc_download_url: "{{ files_repo }}/{{ kube_version }}/runc.{{ image_arch }}"
nerdctl_download_url: "{{ files_repo }}/{{ kube_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
# Insecure registries for containerd
containerd_registries_mirrors:
  - prefix: "{{ registry_addr }}" # https::// 제외
    mirrors:
      - host: "{{ registry_host }}" # https:// 포함
        capabilities: ["pull", "resolve"]
        skip_verify: true # tls 설정 시 해당 부분 삭제

# CentOS/Redhat/AlmaLinux/Rocky Linux
## Docker / Containerd
docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch"
docker_rh_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"

# Fedora
## Docker
docker_fedora_repo_base_url: "{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}"
docker_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
## Containerd
containerd_fedora_repo_base_url: "{{ yum_repo }}/containerd"
containerd_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"

# group_vars/all/all.yml 
apiserver_loadbalancer_domain_name: "" # 3중화 구성시 master LB 주소
loadbalancer_apiserver:
address: "" #  3중화 구성시 master LB 주소
port: 6443

# group_vars/k8s_cluster/k8s_cluster.yml
kube_version: "" # roles/kubespray-defaults/defaults/main/checksums.yml 에서 지원 하는 버전
cluster_name: ""
kube_service_addresses: ""
kube_pods_subnet: ""
service_domain: "" # ingress에 쓰일 domain
rhel_enable_repos: false # public 아니면 false
enalbe_nodelocaldns: false # coredns 와 같이 쓰면 에러 발생

# calico typha 사용 할 시
# group_vars/k8s_cluster/k8s-net-calico.yml
typha_enabled: true
typha_replicas: 3
calico_endpoint_to_host_action: "ACCEPT"
calico_policy_controller_dweployment_nodeselector: "" # node label
policyt_controller_extra_tolerations: "" # taint 값 기입 ex)
  -key: node-role.kubernetes.io/test
   effect: NoSchedule

# group_vars/all/containerd.yml
containerd_registries_mirros:
  - prefix: "example.com" 
    mirrors:
      - host: "https://example.com"
        capabilities: ["pull", "resolve"]

containerd_registry_auth: # imagepullsecrets 이 있기때문에 굳이 사용 안해도 됨.
  - registry: "example.com"
    username: ""  # harbor 같은경우 robot account 사용하면 보안측면에서 사용 가능
    password: ""

#tip
# ansible vars라 아래와 같이 사용도 가능
test1: "waji"
configurations:
  tnrms:
    name: "test"
    autoValue: "1"

## playbook에서 호출 할시
{{ configurations[test1]['name'] }}
{{ configurations[test1]['autoValue'] }}

 

 

← back