Steps to Reproduce (Mantic)

# LXD VM

$ lxc launch --vm --config limits.cpu=2 --config limits.memory=3GiB ubuntu:mantic cinder-mantic
$ lxc shell cinder-mantic

# Apt

apt remove -y unattended-upgrades
apt update

# Network

echo '127.0.0.1 controller' >>/etc/hosts

# MySQL

apt install -y mysql-server python3-pymysql

# RabbitMQ

apt install -y rabbitmq-server

rabbitmqctl add_user openstack RABBIT_PASS
rabbitmqctl set_permissions openstack ".*" ".*" ".*"

# Keystone

cat <<EOF | mysql
CREATE DATABASE keystone;
CREATE USER 'keystone'@'localhost' IDENTIFIED BY 'KEYSTONE_DBPASS';
GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost';
EOF

apt install -y keystone

@ /etc/keystone/keystone.conf

[database]
connection = mysql+pymysql://keystone:KEYSTONE_DBPASS@controller/keystone

[token]
provider = fernet

...

   # mantic has some exceptions ignored in these, but worked fine:
   
su -s /bin/sh -c "keystone-manage db_sync" keystone # exit code 0
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone
keystone-manage bootstrap --bootstrap-password ADMIN_PASS \
  --bootstrap-admin-url http://controller:5000/v3/ \
  --bootstrap-internal-url http://controller:5000/v3/ \
  --bootstrap-public-url http://controller:5000/v3/ \
  --bootstrap-region-id RegionOne
 
echo 'ServerName controller' >>/etc/apache2/apache2.conf
systemctl restart apache2.service

# Openstack Client

cat <<EOF >openstack.rc
export OS_USERNAME=admin
export OS_PASSWORD=ADMIN_PASS
export OS_PROJECT_NAME=admin
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_DOMAIN_NAME=Default
export OS_AUTH_URL=http://controller:5000/v3
export OS_IDENTITY_API_VERSION=3
EOF

source openstack.rc

apt install -y python3-openstackclient

openstack project create --domain default \
  --description "Service Project" service
  
+-------------+----------------------------------+
| Field       | Value                            |
+-------------+----------------------------------+
| description | Service Project                  |
| domain_id   | default                          |
| enabled     | True                             |
| id          | 229ef5671d0b4136b32f1a60584ab725 |
| is_domain   | False                            |
| name        | service                          |
| options     | {}                               |
| parent_id   | default                          |
| tags        | []                               |
+-------------+----------------------------------+

# openstack project list
+----------------------------------+---------+
| ID                               | Name    |
+----------------------------------+---------+
| 0e204837047a4323b8f57ed23bffa4f8 | admin   |
| 229ef5671d0b4136b32f1a60584ab725 | service |
+----------------------------------+---------+

# Cinder

cat <<EOF | mysql
CREATE DATABASE cinder;
CREATE USER 'cinder'@'localhost' IDENTIFIED BY 'CINDER_DBPASS';
GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost';
EOF

openstack user create --domain default --password CINDER_USERPASS cinder
openstack role add --project service --user cinder admin

openstack service create --name cinderv2 \
  --description "OpenStack Block Storage" volumev2
openstack service create --name cinderv3 \
  --description "OpenStack Block Storage" volumev3

openstack endpoint create --region RegionOne \
  volumev2 public http://controller:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev2 internal http://controller:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev2 admin http://controller:8776/v2/%\(project_id\)s

openstack endpoint create --region RegionOne \
  volumev3 public http://controller:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev3 internal http://controller:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev3 admin http://controller:8776/v3/%\(project_id\)s

apt install -y cinder-api cinder-scheduler

@ /etc/cinder/cinder.conf

[DEFAULT]
my_ip = 127.0.0.1
transport_url = rabbit://openstack:RABBIT_PASS@controller
auth_strategy = keystone

[database]
connection = mysql+pymysql://cinder:CINDER_DBPASS@controller/cinder

[keystone_authtoken]
www_authenticate_uri = http://controller:5000
auth_url = http://controller:5000
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = cinder
password = CINDER_USERPASS

[oslo_concurrency]
lock_path = /var/lib/cinder/tmp

...

su -s /bin/sh -c "cinder-manage db sync" cinder

systemctl restart cinder-scheduler.service apache2.service

# Cinder Volume

truncate -s 5GiB /test.img

mknod --mode 0660 /dev/loop-cinder b 7 42
losetup /dev/loop-cinder /test.img

pvcreate /dev/loop-cinder
vgcreate cinder-volumes /dev/loop-cinder

apt install -y cinder-volume tgt lvm2 thin-provisioning-tools

@ /etc/cinder/cinder.conf

[DEFAULT]
enabled_backends = lvm

[lvm]
lvm_type = default
volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
volume_group = cinder-volumes
target_protocol = iscsi
target_helper = tgtadm

...

echo 'include /var/lib/cinder/volumes/*' >/etc/tgt/conf.d/cinder.conf

systemctl restart tgt.service cinder-volume.service


# Cinder Client

apt install -y python3-cinderclient

# cinder list
+----+--------+------+------+----------------+-------------+----------+-------------+
| ID | Status | Name | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+----+--------+------+------+----------------+-------------+----------+-------------+
+----+--------+------+------+----------------+-------------+----------+-------------+

cinder create --name test-volume 1

# cinder list
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status    | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | available | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+

  # mantic requires attachment to set as in-use
  
cinder attachment-create test-volume

# cinder attachment-list
+--------------------------------------+--------------------------------------+----------+-----------+
| ID                                   | Volume ID                            | Status   | Server ID |
+--------------------------------------+--------------------------------------+----------+-----------+
| 90704b06-cf65-473c-adef-7e8c8b5c8b2c | 17c4f736-058b-4cfc-9864-a64ab0995957 | reserved | -         |
+--------------------------------------+--------------------------------------+----------+-----------+

cinder reset-state --state in-use test-volume

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

# Reproducer (15-second startup delay in tgt.service)

FILE=/etc/systemd/system/tgt.service.d/start-delay.conf
mkdir -p $(dirname $FILE)
cat <<EOF > $FILE
[Service]
ExecStartPre=$(which sleep) 15
EOF
systemctl daemon-reload

# date; systemctl restart cinder-volume.service tgt.service; date
Mon Oct 16 21:57:12 UTC 2023
Mon Oct 16 21:57:27 UTC 2023

... Notice that cinder-volume.service is Started _before_ tgt.service

# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
Oct 16 21:57:12 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.
Oct 16 21:57:12 cinder-mantic systemd[1]: Starting tgt.service - (i)SCSI target daemon...
Oct 16 21:57:27 cinder-mantic systemd[1]: Started tgt.service - (i)SCSI target daemon.

# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log
...
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager [None req-ce03264a-6765-41de-8016-a6f27d2685e4 - - - - - -] Failed to re-export volume, setting to ERROR.: oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
...
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager Exit code: 107
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager Stdout: ''
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'
2023-10-16 21:57:18.301 1658 ERROR cinder.volume.manager

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | error  | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

# Undo error state

# systemctl restart cinder-volume.service
# cinder reset-state --state in-use test-volume
# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+





Patched:

# systemctl show cinder-volume.service | grep Wants=
Wants=network-online.target tgt.service

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+


# date; systemctl restart cinder-volume.service tgt.service; date
Wed Oct 18 15:29:01 UTC 2023
Wed Oct 18 15:29:17 UTC 2023

... Notice that cinder-volume.service is Started _after_ tgt.service

# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
Oct 18 15:29:01 cinder-mantic systemd[1]: Starting tgt.service - (i)SCSI target daemon...
Oct 18 15:29:17 cinder-mantic systemd[1]: Started tgt.service - (i)SCSI target daemon.
Oct 18 15:29:17 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+


... Without tgt.service installed:

# apt remove --purge --yes tgt

# systemctl status tgt.service
Unit tgt.service could not be found.

# systemctl show cinder-volume.service | grep Wants=
Wants=tgt.service network-online.target

# systemctl restart cinder-volume.service
# echo $?
0

# systemctl status cinder-volume.service | grep Active:
     Active: active (running) since Wed 2023-10-18 16:47:14 UTC; 11s ago

# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -1
Oct 18 16:47:14 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.