==== start of pod log for container csi-mockplugin-0/csi-provisioner ==== W0411 18:22:42.063134 1 feature_gate.go:237] Setting GA feature gate Topology=true. It will be removed in a future release. I0411 18:22:42.063188 1 feature_gate.go:245] feature gates: &{map[Topology:true]} I0411 18:22:42.063206 1 csi-provisioner.go:150] Version: v3.3.0 I0411 18:22:42.063215 1 csi-provisioner.go:173] Building kube configs for running in cluster... I0411 18:22:42.064441 1 connection.go:154] Connecting to unix:///csi/csi.sock I0411 18:22:43.066246 1 common.go:111] Probing CSI driver for readiness I0411 18:22:43.066280 1 connection.go:183] GRPC call: /csi.v1.Identity/Probe I0411 18:22:43.066290 1 connection.go:184] GRPC request: {} I0411 18:22:43.070981 1 connection.go:186] GRPC response: {} I0411 18:22:43.071054 1 connection.go:187] GRPC error: I0411 18:22:43.071066 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo I0411 18:22:43.071071 1 connection.go:184] GRPC request: {} I0411 18:22:43.071730 1 connection.go:186] GRPC response: {"name":"csi-mock-csi-mock-volumes-8366","vendor_version":"v1.9.0"} I0411 18:22:43.071843 1 connection.go:187] GRPC error: I0411 18:22:43.071865 1 csi-provisioner.go:217] Detected CSI driver csi-mock-csi-mock-volumes-8366 I0411 18:22:43.071889 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginCapabilities I0411 18:22:43.071897 1 connection.go:184] GRPC request: {} I0411 18:22:43.072904 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Service":{"type":1}}},{"Type":{"Service":{"type":2}}}]} I0411 18:22:43.073123 1 connection.go:187] GRPC error: I0411 18:22:43.073141 1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities I0411 18:22:43.073150 1 connection.go:184] GRPC request: {} I0411 18:22:43.074389 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":12}}},{"Type":{"Rpc":{"type":4}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":6}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":11}}},{"Type":{"Rpc":{"type":13}}},{"Type":{"Rpc":{"type":9}}},{"Type":{"Rpc":{"type":2}}}]} I0411 18:22:43.074681 1 connection.go:187] GRPC error: I0411 18:22:43.074806 1 csi-provisioner.go:286] CSI driver supports PUBLISH_UNPUBLISH_VOLUME, watching VolumeAttachments I0411 18:22:43.075828 1 controller.go:732] Using saving PVs to API server in background I0411 18:22:43.078052 1 reflector.go:221] Starting reflector *v1.StorageClass (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078158 1 reflector.go:221] Starting reflector *v1.PersistentVolumeClaim (15m0s) from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078777 1 reflector.go:257] Listing and watching *v1.StorageClass from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078816 1 reflector.go:257] Listing and watching *v1.PersistentVolumeClaim from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078820 1 reflector.go:221] Starting reflector *v1.Node (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078843 1 reflector.go:257] Listing and watching *v1.Node from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078248 1 reflector.go:221] Starting reflector *v1.VolumeAttachment (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078905 1 reflector.go:257] Listing and watching *v1.VolumeAttachment from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078175 1 reflector.go:221] Starting reflector *v1.CSINode (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.078944 1 reflector.go:257] Listing and watching *v1.CSINode from k8s.io/client-go/informers/factory.go:134 I0411 18:22:43.176600 1 shared_informer.go:285] caches populated I0411 18:22:43.176638 1 shared_informer.go:285] caches populated I0411 18:22:43.176666 1 shared_informer.go:285] caches populated I0411 18:22:43.176676 1 shared_informer.go:285] caches populated I0411 18:22:43.176688 1 shared_informer.go:285] caches populated I0411 18:22:43.176704 1 controller.go:811] Starting provisioner controller csi-mock-csi-mock-volumes-8366_csi-mockplugin-0_996c89ac-ec3f-4284-9e5f-5d26c7f1d35d! I0411 18:22:43.176750 1 clone_controller.go:66] Starting CloningProtection controller I0411 18:22:43.176754 1 volume_store.go:97] Starting save volume queue I0411 18:22:43.176886 1 clone_controller.go:82] Started CloningProtection controller I0411 18:22:43.177114 1 reflector.go:221] Starting reflector *v1.StorageClass (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v8/controller/controller.go:848 I0411 18:22:43.177143 1 reflector.go:257] Listing and watching *v1.StorageClass from sigs.k8s.io/sig-storage-lib-external-provisioner/v8/controller/controller.go:848 I0411 18:22:43.177140 1 reflector.go:221] Starting reflector *v1.PersistentVolume (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v8/controller/controller.go:845 I0411 18:22:43.177176 1 reflector.go:257] Listing and watching *v1.PersistentVolume from sigs.k8s.io/sig-storage-lib-external-provisioner/v8/controller/controller.go:845 I0411 18:22:43.276838 1 shared_informer.go:285] caches populated I0411 18:22:43.276914 1 controller.go:860] Started provisioner controller csi-mock-csi-mock-volumes-8366_csi-mockplugin-0_996c89ac-ec3f-4284-9e5f-5d26c7f1d35d! I0411 18:22:50.193085 1 controller.go:1337] provision "csi-mock-volumes-8366/pvc-sc5hc" class "csi-mock-volumes-8366-scbr76s": started I0411 18:22:50.193292 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-8366", Name:"pvc-sc5hc", UID:"c575c950-3a8f-487c-b05f-24af1aefa1a3", APIVersion:"v1", ResourceVersion:"7554524", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "csi-mock-volumes-8366/pvc-sc5hc" I0411 18:22:50.193375 1 connection.go:183] GRPC call: /csi.v1.Controller/CreateVolume I0411 18:22:50.193390 1 connection.go:184] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"topology.hostpath.csi/node":"v126-worker2"}},{"segments":{"topology.hostpath.csi/node":"v126-worker"}}],"requisite":[{"segments":{"topology.hostpath.csi/node":"v126-worker"}},{"segments":{"topology.hostpath.csi/node":"v126-worker2"}}]},"capacity_range":{"required_bytes":1073741824},"name":"pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":7}}]} I0411 18:22:50.198061 1 connection.go:186] GRPC response: {"volume":{"accessible_topology":[{"segments":{"topology.hostpath.csi/node":"v126-worker2"}}],"capacity_bytes":1073741824,"volume_id":"8107e9d9-f830-11ee-809b-7e0a5b4a50df"}} I0411 18:22:50.198247 1 connection.go:187] GRPC error: I0411 18:22:50.198261 1 controller.go:812] create volume rep: {CapacityBytes:1073741824 VolumeId:8107e9d9-f830-11ee-809b-7e0a5b4a50df VolumeContext:map[] ContentSource: AccessibleTopology:[segments: ] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0} I0411 18:22:50.198335 1 controller.go:881] createVolumeOperation: set annotation [volume.kubernetes.io/provisioner-deletion-secret-namespace/volume.kubernetes.io/provisioner-deletion-secret-name] on pv [pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3]. I0411 18:22:50.198364 1 controller.go:906] successfully created PV pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 for PVC pvc-sc5hc and csi volume name 8107e9d9-f830-11ee-809b-7e0a5b4a50df I0411 18:22:50.198386 1 controller.go:922] successfully created PV {GCEPersistentDisk:nil AWSElasticBlockStore:nil HostPath:nil Glusterfs:nil NFS:nil RBD:nil ISCSI:nil Cinder:nil CephFS:nil FC:nil Flocker:nil FlexVolume:nil AzureFile:nil VsphereVolume:nil Quobyte:nil AzureDisk:nil PhotonPersistentDisk:nil PortworxVolume:nil ScaleIO:nil Local:nil StorageOS:nil CSI:&CSIPersistentVolumeSource{Driver:csi-mock-csi-mock-volumes-8366,VolumeHandle:8107e9d9-f830-11ee-809b-7e0a5b4a50df,ReadOnly:false,FSType:ext4,VolumeAttributes:map[string]string{storage.kubernetes.io/csiProvisionerIdentity: 1712859763074-8081-csi-mock-csi-mock-volumes-8366,},ControllerPublishSecretRef:nil,NodeStageSecretRef:nil,NodePublishSecretRef:nil,ControllerExpandSecretRef:nil,NodeExpandSecretRef:nil,}} I0411 18:22:50.198625 1 controller.go:1442] provision "csi-mock-volumes-8366/pvc-sc5hc" class "csi-mock-volumes-8366-scbr76s": volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3" provisioned I0411 18:22:50.198652 1 controller.go:1455] provision "csi-mock-volumes-8366/pvc-sc5hc" class "csi-mock-volumes-8366-scbr76s": succeeded I0411 18:22:50.198665 1 volume_store.go:154] Saving volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 I0411 18:22:50.204724 1 volume_store.go:157] Volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 saved I0411 18:22:50.204789 1 controller.go:1069] Claim processing succeeded, removing PVC c575c950-3a8f-487c-b05f-24af1aefa1a3 from claims in progress I0411 18:22:50.204883 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-8366", Name:"pvc-sc5hc", UID:"c575c950-3a8f-487c-b05f-24af1aefa1a3", APIVersion:"v1", ResourceVersion:"7554524", FieldPath:""}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 I0411 18:23:04.233515 1 controller.go:1337] provision "csi-mock-volumes-8366/pvc-volume-tester-9qmsd-my-volume" class "csi-mock-volumes-8366-scm6zwz": started I0411 18:23:04.233690 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-8366", Name:"pvc-volume-tester-9qmsd-my-volume", UID:"a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851", APIVersion:"v1", ResourceVersion:"7554747", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "csi-mock-volumes-8366/pvc-volume-tester-9qmsd-my-volume" I0411 18:23:04.233914 1 connection.go:183] GRPC call: /csi.v1.Controller/CreateVolume I0411 18:23:04.233932 1 connection.go:184] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"topology.hostpath.csi/node":"v126-worker"}},{"segments":{"topology.hostpath.csi/node":"v126-worker2"}}],"requisite":[{"segments":{"topology.hostpath.csi/node":"v126-worker"}},{"segments":{"topology.hostpath.csi/node":"v126-worker2"}}]},"capacity_range":{"required_bytes":1073741824},"name":"pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":7}}]} I0411 18:23:04.236551 1 connection.go:186] GRPC response: {"volume":{"accessible_topology":[{"segments":{"topology.hostpath.csi/node":"v126-worker2"}}],"capacity_bytes":1073741824,"volume_id":"8966167f-f830-11ee-809b-7e0a5b4a50df"}} I0411 18:23:04.236693 1 connection.go:187] GRPC error: I0411 18:23:04.236708 1 controller.go:812] create volume rep: {CapacityBytes:1073741824 VolumeId:8966167f-f830-11ee-809b-7e0a5b4a50df VolumeContext:map[] ContentSource: AccessibleTopology:[segments: ] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0} I0411 18:23:04.236786 1 controller.go:881] createVolumeOperation: set annotation [volume.kubernetes.io/provisioner-deletion-secret-namespace/volume.kubernetes.io/provisioner-deletion-secret-name] on pv [pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851]. I0411 18:23:04.236836 1 controller.go:906] successfully created PV pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 for PVC pvc-volume-tester-9qmsd-my-volume and csi volume name 8966167f-f830-11ee-809b-7e0a5b4a50df I0411 18:23:04.236879 1 controller.go:922] successfully created PV {GCEPersistentDisk:nil AWSElasticBlockStore:nil HostPath:nil Glusterfs:nil NFS:nil RBD:nil ISCSI:nil Cinder:nil CephFS:nil FC:nil Flocker:nil FlexVolume:nil AzureFile:nil VsphereVolume:nil Quobyte:nil AzureDisk:nil PhotonPersistentDisk:nil PortworxVolume:nil ScaleIO:nil Local:nil StorageOS:nil CSI:&CSIPersistentVolumeSource{Driver:csi-mock-csi-mock-volumes-8366,VolumeHandle:8966167f-f830-11ee-809b-7e0a5b4a50df,ReadOnly:false,FSType:ext4,VolumeAttributes:map[string]string{storage.kubernetes.io/csiProvisionerIdentity: 1712859763074-8081-csi-mock-csi-mock-volumes-8366,},ControllerPublishSecretRef:nil,NodeStageSecretRef:nil,NodePublishSecretRef:nil,ControllerExpandSecretRef:nil,NodeExpandSecretRef:nil,}} I0411 18:23:04.237050 1 controller.go:1442] provision "csi-mock-volumes-8366/pvc-volume-tester-9qmsd-my-volume" class "csi-mock-volumes-8366-scm6zwz": volume "pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851" provisioned I0411 18:23:04.237078 1 controller.go:1455] provision "csi-mock-volumes-8366/pvc-volume-tester-9qmsd-my-volume" class "csi-mock-volumes-8366-scm6zwz": succeeded I0411 18:23:04.237091 1 volume_store.go:154] Saving volume pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 I0411 18:23:04.242018 1 volume_store.go:157] Volume pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 saved I0411 18:23:04.242089 1 controller.go:1069] Claim processing succeeded, removing PVC a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 from claims in progress I0411 18:23:04.242123 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-8366", Name:"pvc-volume-tester-9qmsd-my-volume", UID:"a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851", APIVersion:"v1", ResourceVersion:"7554747", FieldPath:""}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 I0411 18:23:18.285881 1 controller.go:1471] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": started I0411 18:23:18.285976 1 controller.go:1261] volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 does not need any deletion secrets E0411 18:23:18.286026 1 controller.go:1481] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": volume deletion failed: persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 W0411 18:23:18.286109 1 controller.go:989] Retrying syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", failure 0 E0411 18:23:18.286154 1 controller.go:1007] error syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:18.286188 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", UID:"0553763f-a39b-4a06-aa7f-2c7d7e3dd8b5", APIVersion:"v1", ResourceVersion:"7555087", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:18.293903 1 controller.go:1471] delete "pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851": started I0411 18:23:18.293983 1 controller.go:1261] volume pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851 does not need any deletion secrets I0411 18:23:18.294018 1 connection.go:183] GRPC call: /csi.v1.Controller/DeleteVolume I0411 18:23:18.294083 1 connection.go:184] GRPC request: {"volume_id":"8966167f-f830-11ee-809b-7e0a5b4a50df"} I0411 18:23:18.296135 1 connection.go:186] GRPC response: {} I0411 18:23:18.296211 1 connection.go:187] GRPC error: I0411 18:23:18.296230 1 controller.go:1486] delete "pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851": volume deleted I0411 18:23:18.301759 1 controller.go:1531] delete "pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851": persistentvolume deleted I0411 18:23:18.301804 1 controller.go:1536] delete "pvc-a91e7e1b-3bfe-4ec5-b4a5-2e1c9edfa851": succeeded I0411 18:23:19.286624 1 controller.go:1471] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": started I0411 18:23:19.286712 1 controller.go:1261] volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 does not need any deletion secrets E0411 18:23:19.286808 1 controller.go:1481] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": volume deletion failed: persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 W0411 18:23:19.286869 1 controller.go:989] Retrying syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", failure 1 E0411 18:23:19.286905 1 controller.go:1007] error syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:19.286985 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", UID:"0553763f-a39b-4a06-aa7f-2c7d7e3dd8b5", APIVersion:"v1", ResourceVersion:"7555087", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:21.287614 1 controller.go:1471] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": started I0411 18:23:21.287708 1 controller.go:1261] volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 does not need any deletion secrets E0411 18:23:21.287755 1 controller.go:1481] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": volume deletion failed: persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 W0411 18:23:21.287811 1 controller.go:989] Retrying syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", failure 2 E0411 18:23:21.287845 1 controller.go:1007] error syncing volume "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:21.287944 1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3", UID:"0553763f-a39b-4a06-aa7f-2c7d7e3dd8b5", APIVersion:"v1", ResourceVersion:"7555087", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 is still attached to node v126-worker2 I0411 18:23:25.289010 1 controller.go:1471] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": started I0411 18:23:25.289064 1 controller.go:1261] volume pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3 does not need any deletion secrets I0411 18:23:25.289106 1 connection.go:183] GRPC call: /csi.v1.Controller/DeleteVolume I0411 18:23:25.289115 1 connection.go:184] GRPC request: {"volume_id":"8107e9d9-f830-11ee-809b-7e0a5b4a50df"} I0411 18:23:25.290712 1 connection.go:186] GRPC response: {} I0411 18:23:25.290755 1 connection.go:187] GRPC error: I0411 18:23:25.290774 1 controller.go:1486] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": volume deleted I0411 18:23:25.296263 1 controller.go:1531] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": persistentvolume deleted I0411 18:23:25.296305 1 controller.go:1536] delete "pvc-c575c950-3a8f-487c-b05f-24af1aefa1a3": succeeded ==== end of pod log for container csi-mockplugin-0/csi-provisioner ====