==== start of pod log for container csi-mockplugin-0/csi-provisioner ====
W0203 01:05:58.659921       1 feature_gate.go:237] Setting GA feature gate Topology=true. It will be removed in a future release.
I0203 01:05:58.660092       1 feature_gate.go:245] feature gates: &{map[Topology:true]}
I0203 01:05:58.660131       1 csi-provisioner.go:138] Version: v3.0.0
I0203 01:05:58.660162       1 csi-provisioner.go:161] Building kube configs for running in cluster...
I0203 01:05:58.661797       1 connection.go:154] Connecting to unix:///csi/csi.sock
I0203 01:05:59.662955       1 common.go:111] Probing CSI driver for readiness
I0203 01:05:59.662989       1 connection.go:183] GRPC call: /csi.v1.Identity/Probe
I0203 01:05:59.662996       1 connection.go:184] GRPC request: {}
I0203 01:05:59.675848       1 connection.go:186] GRPC response: {}
I0203 01:05:59.675950       1 connection.go:187] GRPC error: <nil>
I0203 01:05:59.675967       1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo
I0203 01:05:59.675981       1 connection.go:184] GRPC request: {}
I0203 01:05:59.677098       1 connection.go:186] GRPC response: {"name":"csi-mock-csi-mock-volumes-3275","vendor_version":"v1.7.3"}
I0203 01:05:59.677193       1 connection.go:187] GRPC error: <nil>
I0203 01:05:59.677233       1 csi-provisioner.go:205] Detected CSI driver csi-mock-csi-mock-volumes-3275
I0203 01:05:59.677252       1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginCapabilities
I0203 01:05:59.677260       1 connection.go:184] GRPC request: {}
I0203 01:05:59.678343       1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Service":{"type":1}}},{"Type":{"Service":{"type":2}}}]}
I0203 01:05:59.678523       1 connection.go:187] GRPC error: <nil>
I0203 01:05:59.678538       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities
I0203 01:05:59.678546       1 connection.go:184] GRPC request: {}
I0203 01:05:59.679690       1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":12}}},{"Type":{"Rpc":{"type":4}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":6}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":11}}},{"Type":{"Rpc":{"type":9}}},{"Type":{"Rpc":{"type":2}}}]}
I0203 01:05:59.680059       1 connection.go:187] GRPC error: <nil>
I0203 01:05:59.680142       1 csi-provisioner.go:274] CSI driver supports PUBLISH_UNPUBLISH_VOLUME, watching VolumeAttachments
I0203 01:05:59.682795       1 controller.go:731] Using saving PVs to API server in background
I0203 01:05:59.683287       1 reflector.go:219] Starting reflector *v1.PersistentVolumeClaim (15m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683320       1 reflector.go:255] Listing and watching *v1.PersistentVolumeClaim from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683355       1 reflector.go:219] Starting reflector *v1.CSINode (1h0m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683380       1 reflector.go:255] Listing and watching *v1.CSINode from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683485       1 reflector.go:219] Starting reflector *v1.StorageClass (1h0m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683510       1 reflector.go:255] Listing and watching *v1.StorageClass from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683559       1 reflector.go:219] Starting reflector *v1.Node (1h0m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683586       1 reflector.go:255] Listing and watching *v1.Node from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683600       1 reflector.go:219] Starting reflector *v1.VolumeAttachment (1h0m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.683627       1 reflector.go:255] Listing and watching *v1.VolumeAttachment from k8s.io/client-go/informers/factory.go:134
I0203 01:05:59.783473       1 shared_informer.go:270] caches populated
I0203 01:05:59.783498       1 shared_informer.go:270] caches populated
I0203 01:05:59.783511       1 shared_informer.go:270] caches populated
I0203 01:05:59.783518       1 shared_informer.go:270] caches populated
I0203 01:05:59.783524       1 shared_informer.go:270] caches populated
I0203 01:05:59.783536       1 controller.go:810] Starting provisioner controller csi-mock-csi-mock-volumes-3275_csi-mockplugin-0_a0fd2e57-ae78-487d-8466-c67f02c11347!
I0203 01:05:59.783565       1 volume_store.go:97] Starting save volume queue
I0203 01:05:59.783569       1 clone_controller.go:66] Starting CloningProtection controller
I0203 01:05:59.783643       1 clone_controller.go:82] Started CloningProtection controller
I0203 01:05:59.783795       1 reflector.go:219] Starting reflector *v1.StorageClass (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:847
I0203 01:05:59.783820       1 reflector.go:255] Listing and watching *v1.StorageClass from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:847
I0203 01:05:59.783894       1 reflector.go:219] Starting reflector *v1.PersistentVolume (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:844
I0203 01:05:59.783921       1 reflector.go:255] Listing and watching *v1.PersistentVolume from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:844
I0203 01:05:59.883992       1 shared_informer.go:270] caches populated
I0203 01:05:59.884054       1 controller.go:859] Started provisioner controller csi-mock-csi-mock-volumes-3275_csi-mockplugin-0_a0fd2e57-ae78-487d-8466-c67f02c11347!
I0203 01:06:02.963751       1 controller.go:1279] provision "csi-mock-volumes-3275/pvc-7pgh4" class "csi-mock-volumes-3275-scf8clk": started
I0203 01:06:02.963917       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-3275", Name:"pvc-7pgh4", UID:"2fae1686-37d2-4f05-82d5-72d6b73feeef", APIVersion:"v1", ResourceVersion:"1462244", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "csi-mock-volumes-3275/pvc-7pgh4"
I0203 01:06:02.964007       1 connection.go:183] GRPC call: /csi.v1.Controller/CreateVolume
I0203 01:06:02.964020       1 connection.go:184] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}},{"segments":{"topology.hostpath.csi/node":"v125-worker"}}],"requisite":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}},{"segments":{"topology.hostpath.csi/node":"v125-worker"}}]},"capacity_range":{"required_bytes":1073741824},"name":"pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
I0203 01:06:02.968972       1 connection.go:186] GRPC response: {"volume":{"accessible_topology":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}}],"capacity_bytes":1073741824,"volume_id":"edc465a0-a35e-11ed-87e3-521beec8a122"}}
I0203 01:06:02.969094       1 connection.go:187] GRPC error: <nil>
I0203 01:06:02.969109       1 controller.go:777] create volume rep: {CapacityBytes:1073741824 VolumeId:edc465a0-a35e-11ed-87e3-521beec8a122 VolumeContext:map[] ContentSource:<nil> AccessibleTopology:[segments:<key:"topology.hostpath.csi/node" value:"v125-worker2" > ] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0203 01:06:02.969177       1 controller.go:861] successfully created PV pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef for PVC pvc-7pgh4 and csi volume name edc465a0-a35e-11ed-87e3-521beec8a122
I0203 01:06:02.969191       1 controller.go:877] successfully created PV {GCEPersistentDisk:nil AWSElasticBlockStore:nil HostPath:nil Glusterfs:nil NFS:nil RBD:nil ISCSI:nil Cinder:nil CephFS:nil FC:nil Flocker:nil FlexVolume:nil AzureFile:nil VsphereVolume:nil Quobyte:nil AzureDisk:nil PhotonPersistentDisk:nil PortworxVolume:nil ScaleIO:nil Local:nil StorageOS:nil CSI:&CSIPersistentVolumeSource{Driver:csi-mock-csi-mock-volumes-3275,VolumeHandle:edc465a0-a35e-11ed-87e3-521beec8a122,ReadOnly:false,FSType:ext4,VolumeAttributes:map[string]string{storage.kubernetes.io/csiProvisionerIdentity: 1675386359680-8081-csi-mock-csi-mock-volumes-3275,},ControllerPublishSecretRef:nil,NodeStageSecretRef:nil,NodePublishSecretRef:nil,ControllerExpandSecretRef:nil,}}
I0203 01:06:02.969329       1 controller.go:1384] provision "csi-mock-volumes-3275/pvc-7pgh4" class "csi-mock-volumes-3275-scf8clk": volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef" provisioned
I0203 01:06:02.969342       1 controller.go:1397] provision "csi-mock-volumes-3275/pvc-7pgh4" class "csi-mock-volumes-3275-scf8clk": succeeded
I0203 01:06:02.969350       1 volume_store.go:154] Saving volume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef
I0203 01:06:02.973308       1 volume_store.go:157] Volume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef saved
I0203 01:06:02.973360       1 controller.go:1068] Claim processing succeeded, removing PVC 2fae1686-37d2-4f05-82d5-72d6b73feeef from claims in progress
I0203 01:06:02.973438       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-3275", Name:"pvc-7pgh4", UID:"2fae1686-37d2-4f05-82d5-72d6b73feeef", APIVersion:"v1", ResourceVersion:"1462244", FieldPath:""}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef
I0203 01:06:15.001370       1 controller.go:1279] provision "csi-mock-volumes-3275/pvc-volume-tester-hx9ck-my-volume" class "csi-mock-volumes-3275-scq2sst": started
I0203 01:06:15.001612       1 connection.go:183] GRPC call: /csi.v1.Controller/CreateVolume
I0203 01:06:15.001599       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-3275", Name:"pvc-volume-tester-hx9ck-my-volume", UID:"99bfaa08-2265-4e53-8979-d7b82eae5b43", APIVersion:"v1", ResourceVersion:"1462645", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "csi-mock-volumes-3275/pvc-volume-tester-hx9ck-my-volume"
I0203 01:06:15.001632       1 connection.go:184] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}},{"segments":{"topology.hostpath.csi/node":"v125-worker"}}],"requisite":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}},{"segments":{"topology.hostpath.csi/node":"v125-worker"}}]},"capacity_range":{"required_bytes":1073741824},"name":"pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
I0203 01:06:15.004351       1 connection.go:186] GRPC response: {"volume":{"accessible_topology":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}}],"capacity_bytes":1073741824,"volume_id":"f4f0cdfd-a35e-11ed-87e3-521beec8a122"}}
I0203 01:06:15.004479       1 connection.go:187] GRPC error: <nil>
I0203 01:06:15.004505       1 controller.go:777] create volume rep: {CapacityBytes:1073741824 VolumeId:f4f0cdfd-a35e-11ed-87e3-521beec8a122 VolumeContext:map[] ContentSource:<nil> AccessibleTopology:[segments:<key:"topology.hostpath.csi/node" value:"v125-worker2" > ] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0203 01:06:15.004582       1 controller.go:861] successfully created PV pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43 for PVC pvc-volume-tester-hx9ck-my-volume and csi volume name f4f0cdfd-a35e-11ed-87e3-521beec8a122
I0203 01:06:15.004601       1 controller.go:877] successfully created PV {GCEPersistentDisk:nil AWSElasticBlockStore:nil HostPath:nil Glusterfs:nil NFS:nil RBD:nil ISCSI:nil Cinder:nil CephFS:nil FC:nil Flocker:nil FlexVolume:nil AzureFile:nil VsphereVolume:nil Quobyte:nil AzureDisk:nil PhotonPersistentDisk:nil PortworxVolume:nil ScaleIO:nil Local:nil StorageOS:nil CSI:&CSIPersistentVolumeSource{Driver:csi-mock-csi-mock-volumes-3275,VolumeHandle:f4f0cdfd-a35e-11ed-87e3-521beec8a122,ReadOnly:false,FSType:ext4,VolumeAttributes:map[string]string{storage.kubernetes.io/csiProvisionerIdentity: 1675386359680-8081-csi-mock-csi-mock-volumes-3275,},ControllerPublishSecretRef:nil,NodeStageSecretRef:nil,NodePublishSecretRef:nil,ControllerExpandSecretRef:nil,}}
I0203 01:06:15.004710       1 controller.go:1384] provision "csi-mock-volumes-3275/pvc-volume-tester-hx9ck-my-volume" class "csi-mock-volumes-3275-scq2sst": volume "pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43" provisioned
I0203 01:06:15.004731       1 controller.go:1397] provision "csi-mock-volumes-3275/pvc-volume-tester-hx9ck-my-volume" class "csi-mock-volumes-3275-scq2sst": succeeded
I0203 01:06:15.004748       1 volume_store.go:154] Saving volume pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43
I0203 01:06:15.009109       1 volume_store.go:157] Volume pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43 saved
I0203 01:06:15.009181       1 controller.go:1068] Claim processing succeeded, removing PVC 99bfaa08-2265-4e53-8979-d7b82eae5b43 from claims in progress
I0203 01:06:15.009421       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-3275", Name:"pvc-volume-tester-hx9ck-my-volume", UID:"99bfaa08-2265-4e53-8979-d7b82eae5b43", APIVersion:"v1", ResourceVersion:"1462645", FieldPath:""}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43
I0203 01:06:27.044770       1 controller.go:1413] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": started
E0203 01:06:27.044939       1 controller.go:1423] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": volume deletion failed: persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
W0203 01:06:27.045009       1 controller.go:988] Retrying syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", failure 0
E0203 01:06:27.045048       1 controller.go:1006] error syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:27.045115       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", UID:"ebab09b8-eaa6-4f33-bead-58feaee69c1d", APIVersion:"v1", ResourceVersion:"1463135", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:27.051486       1 controller.go:1413] delete "pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43": started
I0203 01:06:27.051648       1 connection.go:183] GRPC call: /csi.v1.Controller/DeleteVolume
I0203 01:06:27.051672       1 connection.go:184] GRPC request: {"volume_id":"f4f0cdfd-a35e-11ed-87e3-521beec8a122"}
I0203 01:06:27.053923       1 connection.go:186] GRPC response: {}
I0203 01:06:27.054011       1 connection.go:187] GRPC error: <nil>
I0203 01:06:27.054025       1 controller.go:1428] delete "pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43": volume deleted
I0203 01:06:27.058220       1 controller.go:1478] delete "pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43": persistentvolume deleted
I0203 01:06:27.058255       1 controller.go:1483] delete "pvc-99bfaa08-2265-4e53-8979-d7b82eae5b43": succeeded
I0203 01:06:28.045309       1 controller.go:1413] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": started
E0203 01:06:28.045481       1 controller.go:1423] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": volume deletion failed: persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
W0203 01:06:28.045555       1 controller.go:988] Retrying syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", failure 1
E0203 01:06:28.045604       1 controller.go:1006] error syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:28.045667       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", UID:"ebab09b8-eaa6-4f33-bead-58feaee69c1d", APIVersion:"v1", ResourceVersion:"1463135", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:30.046333       1 controller.go:1413] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": started
E0203 01:06:30.046448       1 controller.go:1423] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": volume deletion failed: persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
W0203 01:06:30.046519       1 controller.go:988] Retrying syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", failure 2
E0203 01:06:30.046562       1 controller.go:1006] error syncing volume "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:30.046597       1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef", UID:"ebab09b8-eaa6-4f33-bead-58feaee69c1d", APIVersion:"v1", ResourceVersion:"1463135", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef is still attached to node v125-worker2
I0203 01:06:34.047380       1 controller.go:1413] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": started
I0203 01:06:34.047517       1 connection.go:183] GRPC call: /csi.v1.Controller/DeleteVolume
I0203 01:06:34.047533       1 connection.go:184] GRPC request: {"volume_id":"edc465a0-a35e-11ed-87e3-521beec8a122"}
I0203 01:06:34.048818       1 connection.go:186] GRPC response: {}
I0203 01:06:34.048860       1 connection.go:187] GRPC error: <nil>
I0203 01:06:34.048875       1 controller.go:1428] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": volume deleted
I0203 01:06:34.052636       1 controller.go:1478] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": persistentvolume deleted
I0203 01:06:34.052662       1 controller.go:1483] delete "pvc-2fae1686-37d2-4f05-82d5-72d6b73feeef": succeeded
==== end of pod log for container csi-mockplugin-0/csi-provisioner ====