hypershift: extra_pools

karmab · Aug 5, 2024 · c11d093 · c11d093
1 parent 515de6e
commit c11d093
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 66 deletions.
diff --git a/kvirt/cluster/hypershift/__init__.py b/kvirt/cluster/hypershift/__init__.py
@@ -171,6 +171,47 @@ def handle_baremetal_iso(config, plandir, cluster, data, baremetal_hosts=[]):
         return f'http://{svcip}:{svcport}/{cluster}-worker.iso'
 
 
+def process_nodepools(config, plandir, namespace, cluster, platform, nodepools, nodepool_image, manifests, overrides):
+    clusterdir = os.path.expanduser(f"~/.kcli/clusters/{cluster}")
+    nodepools = [{'name': entry} if isinstance(entry, str) else entry for entry in nodepools]
+    nodepool_data = {'nodepools': nodepools, 'namespace': namespace, 'cluster': cluster,
+                     'nodepool_image': nodepool_image, 'platform': platform, 'workers': overrides.get('workers', 2)}
+    kubevirt_data = {'numcpus': overrides.get('numcpus', 8), 'memory': overrides.get('memory', 6144),
+                     'disk_size': overrides.get('disk_size', 30)}
+    nodepool_data.update(kubevirt_data)
+    if manifests:
+        nodepool_data['manifests'] = manifests
+    nodepoolfile = config.process_inputfile(cluster, f"{plandir}/nodepools.yaml", overrides=nodepool_data)
+    with open(f"{clusterdir}/nodepools.yaml", 'w') as f:
+        f.write(nodepoolfile)
+    cmcmd = f"oc create -f {clusterdir}/nodepools.yaml"
+    call(cmcmd, shell=True)
+    if platform is not None:
+        return
+    for entry in nodepools:
+        nodepool = entry['name']
+        pprint(f"Waiting before ignition data for nodepool {nodepool} is available")
+        user_data = f"user-data-{cluster}"
+        cmd = f"until oc -n {namespace}-{cluster} get secret | grep {user_data} >/dev/null 2>&1 ; do sleep 1 ; done"
+        call(cmd, shell=True)
+        ignition_data = {'namespace': namespace, 'cluster': cluster, 'nodepool': nodepool}
+        ignitionscript = config.process_inputfile(cluster, f"{plandir}/ignition.sh", overrides=ignition_data)
+        with open(f"{clusterdir}/ignition_{nodepool}.sh", 'w') as f:
+            f.write(ignitionscript)
+        ignition_worker = f"{clusterdir}/nodepool_{nodepool}.ign"
+        timeout = 0
+        while True:
+            if os.path.exists(ignition_worker):
+                break
+            else:
+                sleep(30)
+                timeout += 30
+                if timeout > 300:
+                    msg = "Timeout trying to retrieve worker ignition"
+                    return {'result': 'failure', 'reason': msg}
+            call(f'bash {clusterdir}/ignition_{nodepool}.sh', shell=True)
+
+
 def scale(config, plandir, cluster, overrides):
     storedparameters = overrides.get('storedparameters', True)
     provider = config.type
@@ -208,6 +249,7 @@ def scale(config, plandir, cluster, overrides):
             plan = installparam.get('plan', plan)
             old_assisted_vms_number = installparam.get('assisted_vms_number', old_assisted_vms_number)
     data.update(overrides)
+    nodepool = overrides.get('nodepool') or cluster
     if os.path.exists(f"{clusterdir}/kubeconfig.mgmt"):
         os.environ['KUBECONFIG'] = f"{clusterdir}/kubeconfig.mgmt"
     if 'KUBECONFIG' not in os.environ:
@@ -222,7 +264,7 @@ def scale(config, plandir, cluster, overrides):
     worker_overrides = data.copy()
     workers = worker_overrides.get('workers', 2)
     if kubevirt:
-        cmcmd = f"oc -n {namespace}-{cluster} scale nodepool {cluster} --replicas {workers}"
+        cmcmd = f"oc -n {namespace}-{cluster} scale nodepool {nodepool} --replicas {workers}"
         call(cmcmd, shell=True)
         return {'result': 'success'}
     os.chdir(os.path.expanduser("~/.kcli"))
@@ -257,7 +299,7 @@ def scale(config, plandir, cluster, overrides):
         if assisted:
             all_baremetal_hosts = old_baremetal_hosts + baremetal_hosts
             create_bmh_objects(config, plandir, cluster, namespace, all_baremetal_hosts, overrides)
-            cmcmd = f"oc -n {data['namespace']} scale nodepool {cluster} --replicas {len(all_baremetal_hosts)}"
+            cmcmd = f"oc -n {data['namespace']} scale nodepool {nodepool} --replicas {len(all_baremetal_hosts)}"
             call(cmcmd, shell=True)
             return {'result': 'success'}
         else:
@@ -590,8 +632,6 @@ def create(config, plandir, cluster, overrides):
         assisted_data = config.process_inputfile(cluster, f'{plandir}/assisted_ingress.yml', overrides=assisted_data)
         assisted_data = json.dumps(assisted_data)
         manifests.append({'name': f'assisted-ingress-{cluster}', 'data': assisted_data})
-    if manifests:
-        assetsdata['manifests'] = manifests
     async_files = []
     async_tempdir = TemporaryDirectory()
     asyncdir = async_tempdir.name
@@ -714,10 +754,9 @@ def create(config, plandir, cluster, overrides):
     INSTALLER_VERSION = get_installer_version()
     pprint(f"Using installer version {INSTALLER_VERSION}")
     if 'OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE' in os.environ:
-        assetsdata['nodepool_image'] = os.environ['OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE']
+        nodepool_image = os.environ['OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE']
     else:
         nodepool_image = os.popen("openshift-install version | grep 'release image' | cut -f3 -d' '").read().strip()
-        assetsdata['nodepool_image'] = nodepool_image
     if assisted:
         if assisted_vms:
             if data['disk_size'] < 200:
@@ -799,32 +838,8 @@ def create(config, plandir, cluster, overrides):
             installparam['baremetal_hosts'] = baremetal_hosts
         installparam['assisted_vms_number'] = data['assisted_vms_number']
         safe_dump(installparam, p, default_flow_style=False, encoding='utf-8', allow_unicode=True)
-    nodepoolfile = config.process_inputfile(cluster, f"{plandir}/nodepool.yaml", overrides=assetsdata)
-    with open(f"{clusterdir}/nodepool.yaml", 'w') as f:
-        f.write(nodepoolfile)
-    cmcmd = f"oc create -f {clusterdir}/nodepool.yaml"
-    call(cmcmd, shell=True)
-    assetsdata['clusterdir'] = clusterdir
-    if platform is None:
-        ignitionscript = config.process_inputfile(cluster, f"{plandir}/ignition.sh", overrides=assetsdata)
-        with open(f"{clusterdir}/ignition.sh", 'w') as f:
-            f.write(ignitionscript)
-        pprint("Waiting before ignition data is available")
-        user_data = f"user-data-{cluster}"
-        call(f"until oc -n {namespace}-{cluster} get secret | grep {user_data} >/dev/null 2>&1 ; do sleep 1 ; done",
-             shell=True)
-        ignition_worker = f"{clusterdir}/nodepool.ign"
-        timeout = 0
-        while True:
-            if os.path.exists(ignition_worker):
-                break
-            else:
-                sleep(30)
-                timeout += 30
-                if timeout > 300:
-                    msg = "Timeout trying to retrieve worker ignition"
-                    return {'result': 'failure', 'reason': msg}
-            call(f'bash {clusterdir}/ignition.sh', shell=True)
+    nodepools = [cluster] + data.get('extra_nodepools', [])
+    process_nodepools(config, plandir, namespace, cluster, platform, nodepools, nodepool_image, manifests, overrides)
     if 'name' in data:
         del data['name']
     autoapproverpath = f'{clusterdir}/autoapprovercron.yml'
@@ -834,14 +849,14 @@ def create(config, plandir, cluster, overrides):
     call(f"oc apply -f {autoapproverpath}", shell=True)
     if platform is None:
         if provider in cloud_providers + ['openstack']:
-            copy2(f"{clusterdir}/nodepool.ign", f"{clusterdir}/nodepool.ign.ori")
+            copy2(f"{clusterdir}/nodepool_{cluster}.ign", f"{clusterdir}/nodepool_{cluster}.ign.ori")
             bucket = f"{cluster}-{domain.replace('.', '-')}"
             if bucket not in config.k.list_buckets():
                 config.k.create_bucket(bucket)
-            config.k.upload_to_bucket(bucket, f"{clusterdir}/nodepool.ign", public=True)
-            bucket_url = config.k.public_bucketfile_url(bucket, "nodepool.ign")
+            config.k.upload_to_bucket(bucket, f"{clusterdir}/nodepool_{cluster}.ign", public=True)
+            bucket_url = config.k.public_bucketfile_url(bucket, "nodepool_{cluster}.ign")
             new_ignition = {'ignition': {'config': {'merge': [{'source': bucket_url}]}, 'version': '3.2.0'}}
-            with open(f"{clusterdir}/nodepool.ign", 'w') as f:
+            with open(f"{clusterdir}/nodepool_{cluster}.ign", 'w') as f:
                 f.write(json.dumps(new_ignition))
         elif baremetal_iso or baremetal_hosts:
             iso_url = handle_baremetal_iso(config, plandir, cluster, data, baremetal_hosts)

diff --git a/kvirt/cluster/hypershift/ignition.sh b/kvirt/cluster/hypershift/ignition.sh
@@ -1,20 +1,12 @@
 #!/bin/bash
 
-NAMESPACE={{ namespace }}-{{ cluster }}
+BASE={{ namespace }}
 CLUSTER={{ cluster }}
-CLUSTERDIR={{ clusterdir }} 
-SECRET=$(oc -n $NAMESPACE get secret | grep user-data-$CLUSTER | head -1 | awk '{print $1}')
-TOKEN=$(oc -n $NAMESPACE get secret $SECRET -o jsonpath='{.data.value}' | base64 -d | awk -F "Bearer " '{print $2}' | awk -F "\"" '{print "Bearer " $1}')
+NODEPOOL={{ nodepool|default(cluster) }}
+NAMESPACE=${BASE}-${CLUSTER}
+CLUSTERDIR=$HOME/.kcli/clusters/$CLUSTER
 
-{% if nodeport|default(False) %}
-IP=$(oc get node -o wide --selector='node-role.kubernetes.io/master' | grep -v NAME|  head -1 | awk '{print $6}')
-PORT=$(oc -n $NAMESPACE get svc ignition-server-proxy -o jsonpath={.spec.ports[0].nodePort})
-curl -k -H "Authorization: $TOKEN" https://$IP:$PORT/ignition > $CLUSTERDIR/nodepool.ign
-{% else %}
-MANAGEMENT_INGRESS_DOMAIN={{ management_ingress_domain }}
-curl -k -H "Authorization: $TOKEN" https://ignition-server-$NAMESPACE.$MANAGEMENT_INGRESS_DOMAIN/ignition > $CLUSTERDIR/nodepool.ign
-{% endif %}
+SECRET=$(oc -n $NAMESPACE get secret -o jsonpath="{range .items[?(@.metadata.annotations.hypershift\.openshift\.io/nodePool==\"$BASE/$NODEPOOL\")]}{.metadata.name}{'\n'}{end}" | grep token)
+oc -n $NAMESPACE get secret $SECRET -o jsonpath={'.data.payload'} | base64 -d > $CLUSTERDIR/nodepool_$NODEPOOL.ign
 
-if [ ! -s $CLUSTERDIR/nodepool.ign ] || [ "$(grep 'Token not found' $CLUSTERDIR/nodepool.ign)" != "" ] || [ "$(grep '503 Service Unavailable' $CLUSTERDIR/nodepool.ign)" != "" ] ; then
-  rm -f $CLUSTERDIR/nodepool.ign
-fi
+[ -s $CLUSTERDIR/nodepool_$NODEPOOL.ign ] || rm -f $CLUSTERDIR/nodepool_$NODEPOOL.ign
diff --git a/kvirt/cluster/hypershift/kcli_default.yml b/kvirt/cluster/hypershift/kcli_default.yml
@@ -84,3 +84,4 @@ storedparameters: true
 assisted_vms: False
 assisted_vms_number: 2
 assisted_vms_ksushy_ip: 192.168.122.1
+extra_nodepools: []
diff --git a/kvirt/cluster/hypershift/kcli_plan.yml b/kvirt/cluster/hypershift/kcli_plan.yml
@@ -48,6 +48,7 @@
  domain: {{ original_domain|default('karmalabs.corp') }}
  image: {{ image }}
  pool: {{ pool or config_pool }}
+ nodepool: {{ nodepool|default(cluster) }}
  notify: false
  keys: {{ keys }}
  flavor: {{ flavor }}

diff --git a/kvirt/cluster/hypershift/nodepool.yaml → kvirt/cluster/hypershift/nodepools.yaml b/kvirt/cluster/hypershift/nodepool.yaml → kvirt/cluster/hypershift/nodepools.yaml
@@ -11,18 +11,19 @@ data:
 {% endfor %}
 {% endif %}
 ---
+{% for nodepool in nodepools %}
 apiVersion: hypershift.openshift.io/v1beta1
 kind: NodePool
 metadata:
-  name: {{ cluster }}
+  name: {{ nodepool['name'] }}
   namespace: {{ namespace }}
 spec:
   clusterName: {{ cluster }}
   management:
     autoRepair: false
     upgradeType: InPlace
   platform:
-{% if kubevirt %}
+{% if platform == 'kubevirt' %}
     kubevirt:
       compute:
         cores: {{ numcpus|default(8) }}
@@ -33,12 +34,12 @@ spec:
         type: Persistent
     type: KubeVirt
 {% else %}
-    type: {{ 'Agent' if assisted else 'None' }}
+    type: {{ 'Agent' if platform == 'assisted' else 'None' }}
 {% endif %}
   release:
-    image: "{{ nodepool_image }}"
+    image: {{ nodepool['image']|default(nodepool_image) }}
   replicas: {{ workers }}
-{% if manifests|default([]) %}
+{% if loop.index0 == 0 and manifests|default([]) %}
   config:
 {% for entry in manifests %}
   - name: {{ entry['name'] }}
@@ -47,3 +48,5 @@ spec:
 status:
   conditions: null
   replicas: {{ workers }}
+---
+{% endfor %}
diff --git a/kvirt/cluster/hypershift/staticpods/coredns.yml b/kvirt/cluster/hypershift/staticpods/coredns.yml
@@ -17,7 +17,7 @@ spec:
   - name: update-coredns-conf
     securityContext:
       privileged: true
-    image: {{ registry }}/karmab/curl:{{ arch_tag }}
+    image: {{ registry|default('quay.io') }}/karmab/curl:{{ arch_tag }}
     command:
     - "/bin/sh"
     - "-c"
@@ -42,7 +42,7 @@ spec:
   - name: coredns
     securityContext:
       privileged: true
-    image: {{ registry }}/karmab/origin-coredns:{{ arch_tag }}
+    image: {{ registry|default('quay.io') }}/karmab/origin-coredns:{{ arch_tag }}
     args:
     - "--conf"
     - "/etc/coredns/Corefile"

diff --git a/kvirt/cluster/hypershift/staticpods/keepalived.yml b/kvirt/cluster/hypershift/staticpods/keepalived.yml
@@ -17,7 +17,7 @@ spec:
   - name: fix-keepalived-conf
     securityContext:
       privileged: true
-    image: {{ registry }}/karmab/curl:{{ arch_tag }}
+    image: {{ registry|default('quay.io') }}/karmab/curl:{{ arch_tag }}
     command:
     - "/bin/sh"
     - "-c"
@@ -49,7 +49,7 @@ spec:
   - name: keepalived
     securityContext:
       privileged: true
-    image: {{ registry }}/karmab/origin-keepalived-ipfailover:{{ arch_tag }}
+    image: {{ registry|default('quay.io') }}/karmab/origin-keepalived-ipfailover:{{ arch_tag }}
     command:
     - /usr/sbin/keepalived
     args:

diff --git a/kvirt/cluster/hypershift/staticpods/mdns.yml b/kvirt/cluster/hypershift/staticpods/mdns.yml
@@ -17,7 +17,7 @@ spec:
   - name: mdns-render-config
     securityContext:
       privileged: true
-    image: {{ registry }}/karmab/curl:latest
+    image: {{ registry|default('quay.io') }}/karmab/curl:latest
     command:
     - "/bin/sh"
     - "-c"
@@ -41,7 +41,7 @@ spec:
     imagePullPolicy: IfNotPresent
   containers:
   - name: mdns-publish
-    image: {{ registry }}/karmab/mdns-publisher:latest
+    image: {{ registry|default('quay.io') }}/karmab/mdns-publisher:latest
     args:
     - "--debug"
     resources:

diff --git a/kvirt/common/__init__.py b/kvirt/common/__init__.py
@@ -1200,7 +1200,8 @@ def ignition(name, keys=[], cmds=[], nets=[], gateway=None, dns=None, domain=Non
         role = name.split('-')[-1]
     if role is not None:
         cluster = overrides.get('cluster', plan)
-        ignitionclusterpath = find_ignition_files(role, cluster=cluster)
+        nodepool = overrides.get('nodepool')
+        ignitionclusterpath = find_ignition_files(role, cluster, nodepool)
         if ignitionclusterpath is not None:
             data = mergeignition(name, ignitionclusterpath, data)
         rolepath = f"/workdir/{plan}-{role}.ign" if container_mode() else f"{plan}-{role}.ign"
@@ -1370,12 +1371,13 @@ def get_latest_fedora(arch='x86_64'):
             return f"https://download.fedoraproject.org/pub/fedora/linux/releases/{major}/Cloud/{arch}/images/{short}"
 
 
-def find_ignition_files(role, cluster):
+def find_ignition_files(role, cluster, nodepool):
     clusterpath = os.path.expanduser(f"~/.kcli/clusters/{cluster}/{role}.ign")
-    nodepoolpath = os.path.expanduser(f"~/.kcli/clusters/{cluster}/nodepool.ign")
+    nodepool = nodepool or cluster
+    nodepoolpath = os.path.expanduser(f"~/.kcli/clusters/{cluster}/nodepool_{nodepool}.ign")
     if os.path.exists(clusterpath):
         return clusterpath
-    elif os.path.exists(nodepoolpath):
+    elif nodepoolpath is not None and os.path.exists(nodepoolpath):
         return nodepoolpath
     else:
         return None