Comment 19 for bug 1989686

##kinetic validation 22.3.4

I was able to validate kinetic failure path 22.3.3 using this procedure below and success across upgrade to 22.3.4:

$ wget https://cloud-images.ubuntu.com/kinetic/20220925/kinetic-server-cloudimg-amd64.img
$ oci os object put -ns intcanonical -bn blackboxsw-bucket --file kinetic-server-cloudimg-amd64.img
$ oci compute image import from-object-uri --display-name="blackboxsw-test-kinetic" --uri https://objectstorage.us-phoenix-1.oraclecloud.com/n/intcanonical/b/blackboxsw-bucket/o/kinetic-server-cloudimg-amd64.img --compartment-id="ocid1.compartment.oc1..aaaaaaaagydpgegyncqufyw45spizdp7t4bxpy752csiioelfyyykvedkf4a" --launch-mode="PARAVIRTUALIZED" --source-image-type="qcow2"

# Launch with cloud-config setting bogus passw0rd because we expect to have to use serial console to login
$ cat > set-hostname.yaml <<EOF
## template: jinja
#cloud-config
ssh_import_id: [chad.smith]
hostname: SRU-worked-{{v1.cloud_name}}
password: passw0rd
chpasswd: { expire: False }
ssh_pwauth: True
EOF

# Launch instance with poor default password as we expect network to come up broken

$ oci compute instance launch --availability-domain="qIZq:PHX-AD-3" --shape="VM.Standard2.1" --image-id="ocid1.image.oc1.phx.aaaaaaaazgo7n42d6zl4ucig4w5spqnmaexakaqc6trsukgbxaslynrsvqsq" --subnet-id="ocid1.subnet.oc1.phx.aaaaaaaacosethjtrcdct2imjdbrsruvfhhiaq2xf64d4klhuy54aflokqaq" --ssh-authorized-keys-file=".ssh/id_rsa.pub" --user-data-file="./sethostname.yaml" -c ocid1.tenancy.oc1..aaaaaaaao7f7cccogqrg5emjxkxmctzbnhl6zdkkx36yq2jgxnm4p5vmysbq --display-name bbsw-kinetic-dbg

# Run sample script from serial console to capture current broken state
cat oracle-sru-verification.sh
#!/bin/bash
set -ex
echo "===== confirm failure"
dpkg-query --show cloud-init
ip a
ip r
cat /etc/netplan/50-cloud-init.yaml
for file in /run/systemd/network/*; do
 cat $file;
done
egrep 'WARNING|Applying' /var/log/cloud-init.log

echo "===== fix network manually"
MAC=$(cat /sys/class/net/ens3/address)
cat > 50-cloud-init.yaml <<EOF
# Temporarily fix netplan setting dhcp on primary NIC
network:
    version: 2
    ethernets:
        ens3:
            dhcp4: true
            match:
                macaddress: $MAC
            set-name: ens3
EOF
sudo cp 50-cloud-init.yaml /etc/netplan
sudo netplan generate
sudo netplan apply
echo "===== confirm success across upgrade/reboot"
sudo add-apt-repository ppa:cloud-init-dev/proposed -y
sudo apt install cloud-init -y
sudo rm /etc/netplan/50-cloud-init.yaml
sudo cloud-init clean --logs --reboot

$ ssh ubuntu@VM_IP # Assume we can now get in (a measure of success)

# Check prior script run in failed config case

+ echo '===== confirm failure'
===== confirm failure
+ dpkg-query --show cloud-init
cloud-init 22.3.3-0ubuntu1~22.10.1
+ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 02:00:17:0d:c2:d6 brd ff:ff:ff:ff:ff:ff
    altname enp0s3
    inet 10.0.0.130/24 brd 10.0.0.255 scope global ens3
       valid_lft forever preferred_lft forever
    inet6 fe80::17ff:fe0d:c2d6/64 scope link
       valid_lft forever preferred_lft forever
+ ip r
10.0.0.0/24 dev ens3 proto kernel scope link src 10.0.0.130
+ cat /etc/netplan/50-cloud-init.yaml
# This file is generated from information provided by the datasource. Changes
# to it will not persist across an instance reboot. To disable cloud-init's
# network configuration capabilities, write a file
# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
# network: {config: disabled}
network:
    version: 2
    ethernets:
        ens3:
            addresses:
            - 10.0.0.130/24
            match:
                macaddress: 02:00:17:0d:c2:d6
            mtu: 9000
            set-name: ens3
+ for file in /run/systemd/network/*
+ cat /run/systemd/network/10-netplan-ens3.link
[Match]
MACAddress=02:00:17:0d:c2:d6

[Link]
Name=ens3
WakeOnLan=off
MTUBytes=9000
+ for file in /run/systemd/network/*
+ cat /run/systemd/network/10-netplan-ens3.network
[Match]
MACAddress=02:00:17:0d:c2:d6
Name=ens3

[Link]
MTUBytes=9000

[Network]
LinkLocalAddressing=ipv6
Address=10.0.0.130/24
+ egrep 'WARNING|Applying' /var/log/cloud-init.log
2022-10-06 22:36:49,941 - DataSourceOracle.py[WARNING]: Could not obtain network configuration from initramfs. Falling back to IMDS.
2022-10-06 22:36:49,980 - stages.py[INFO]: Applying network configuration from ds bringup=False: {'config': [{'name': 'ens3', 'type': 'physical', 'mac_address': '02:00:17:0d:c2:d6', 'mtu': 9000, 'subnets': [{'type': 'static', 'address': '10.0.0.130/24'}]}], 'version': 1}
2022-10-06 22:37:04,789 - DataSourceOracle.py[WARNING]: Could not obtain network configuration from initramfs. Falling back to IMDS.
2022-10-06 22:37:43,291 - util.py[WARNING]: Failed to run command to import ubuntu SSH ids
2022-10-06 22:37:43,295 - util.py[WARNING]: ssh-import-id failed for: ubuntu ['chad.smith']
2022-10-06 22:37:43,298 - util.py[WARNING]: Running module ssh-import-id (<module 'cloudinit.config.cc_ssh_import_id' from '/usr/lib/python3/dist-packages/cloudinit/config/cc_ssh_import_id.py'>) failed

##### Validate no errors and proper routes/ip/config on 22.3.4
ubuntu@SRU-worked-oracle:~$ cloud-init --version
/usr/bin/cloud-init 22.3.4-0ubuntu1
ubuntu@SRU-worked-oracle:~$ cloud-init status --long --wait

status: done
time: Fri, 07 Oct 2022 02:27:32 +0000
detail:
DataSourceOracle
ubuntu@SRU-worked-oracle:~$ egrep 'Trace|WARN' /var/log/cloud-init.log
2022-10-07 02:27:20,853 - DataSourceOracle.py[WARNING]: Could not obtain network configuration from initramfs. Falling back to IMDS.
2022-10-07 02:27:23,110 - DataSourceOracle.py[WARNING]: Could not obtain network configuration from initramfs. Falling back to IMDS.
ubuntu@SRU-worked-oracle:~$ cat /etc/netplan/50-cloud-init.yaml
# This file is generated from information provided by the datasource. Changes
# to it will not persist across an instance reboot. To disable cloud-init's
# network configuration capabilities, write a file
# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
# network: {config: disabled}
network:
    version: 2
    ethernets:
        ens3:
            dhcp4: true
            match:
                macaddress: 02:00:17:0d:c2:d6
            mtu: 9000
            set-name: ens3
ubuntu@SRU-worked-oracle:~$ ip r
default via 10.0.0.1 dev ens3 proto dhcp src 10.0.0.130 metric 100
10.0.0.0/24 dev ens3 proto kernel scope link src 10.0.0.130 metric 100
10.0.0.1 dev ens3 proto dhcp scope link src 10.0.0.130 metric 100
169.254.169.254 via 10.0.0.1 dev ens3 proto dhcp src 10.0.0.130 metric 100
ubuntu@SRU-worked-oracle:~$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 02:00:17:0d:c2:d6 brd ff:ff:ff:ff:ff:ff
    altname enp0s3
    inet 10.0.0.130/24 metric 100 brd 10.0.0.255 scope global dynamic ens3
       valid_lft 86348sec preferred_lft 86348sec
    inet6 fe80::17ff:fe0d:c2d6/64 scope link
       valid_lft forever preferred_lft forever