==== start of pod log for container csi-mockplugin-0/csi-provisioner ==== W0505 01:20:57.654359 1 feature_gate.go:237] Setting GA feature gate Topology=true. It will be removed in a future release. I0505 01:20:57.654410 1 feature_gate.go:245] feature gates: &{map[Topology:true]} I0505 01:20:57.654441 1 csi-provisioner.go:138] Version: v3.0.0 I0505 01:20:57.654473 1 csi-provisioner.go:161] Building kube configs for running in cluster... I0505 01:20:57.656158 1 connection.go:154] Connecting to unix:///csi/csi.sock I0505 01:20:58.657324 1 common.go:111] Probing CSI driver for readiness I0505 01:20:58.657355 1 connection.go:183] GRPC call: /csi.v1.Identity/Probe I0505 01:20:58.657363 1 connection.go:184] GRPC request: {} I0505 01:20:58.666551 1 connection.go:186] GRPC response: {} I0505 01:20:58.666741 1 connection.go:187] GRPC error: I0505 01:20:58.666781 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo I0505 01:20:58.666805 1 connection.go:184] GRPC request: {} I0505 01:20:58.667569 1 connection.go:186] GRPC response: {"name":"csi-mock-csi-mock-volumes-775","vendor_version":"v1.7.3"} I0505 01:20:58.667698 1 connection.go:187] GRPC error: I0505 01:20:58.667721 1 csi-provisioner.go:205] Detected CSI driver csi-mock-csi-mock-volumes-775 I0505 01:20:58.667739 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginCapabilities I0505 01:20:58.667748 1 connection.go:184] GRPC request: {} I0505 01:20:58.668477 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Service":{"type":1}}},{"Type":{"Service":{"type":2}}}]} I0505 01:20:58.668716 1 connection.go:187] GRPC error: I0505 01:20:58.668734 1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities I0505 01:20:58.668742 1 connection.go:184] GRPC request: {} I0505 01:20:58.669575 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":12}}},{"Type":{"Rpc":{"type":4}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":6}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":11}}},{"Type":{"Rpc":{"type":9}}},{"Type":{"Rpc":{"type":2}}}]} I0505 01:20:58.669883 1 connection.go:187] GRPC error: I0505 01:20:58.669976 1 csi-provisioner.go:274] CSI driver supports PUBLISH_UNPUBLISH_VOLUME, watching VolumeAttachments I0505 01:20:58.671081 1 controller.go:731] Using saving PVs to API server in background I0505 01:20:58.671472 1 reflector.go:219] Starting reflector *v1.CSINode (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671499 1 reflector.go:255] Listing and watching *v1.CSINode from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671626 1 reflector.go:219] Starting reflector *v1.Node (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671645 1 reflector.go:255] Listing and watching *v1.Node from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671677 1 reflector.go:219] Starting reflector *v1.StorageClass (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671704 1 reflector.go:255] Listing and watching *v1.StorageClass from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671743 1 reflector.go:219] Starting reflector *v1.PersistentVolumeClaim (15m0s) from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671795 1 reflector.go:255] Listing and watching *v1.PersistentVolumeClaim from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671901 1 reflector.go:219] Starting reflector *v1.VolumeAttachment (1h0m0s) from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.671932 1 reflector.go:255] Listing and watching *v1.VolumeAttachment from k8s.io/client-go/informers/factory.go:134 I0505 01:20:58.771899 1 shared_informer.go:270] caches populated I0505 01:20:58.771930 1 shared_informer.go:270] caches populated I0505 01:20:58.771943 1 shared_informer.go:270] caches populated I0505 01:20:58.771952 1 shared_informer.go:270] caches populated I0505 01:20:58.771962 1 shared_informer.go:270] caches populated I0505 01:20:58.771978 1 controller.go:810] Starting provisioner controller csi-mock-csi-mock-volumes-775_csi-mockplugin-0_b5d74148-b736-4055-802f-d71214094e4d! I0505 01:20:58.772003 1 clone_controller.go:66] Starting CloningProtection controller I0505 01:20:58.772046 1 volume_store.go:97] Starting save volume queue I0505 01:20:58.772116 1 clone_controller.go:82] Started CloningProtection controller I0505 01:20:58.772405 1 reflector.go:219] Starting reflector *v1.StorageClass (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:847 I0505 01:20:58.772424 1 reflector.go:255] Listing and watching *v1.StorageClass from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:847 I0505 01:20:58.772471 1 reflector.go:219] Starting reflector *v1.PersistentVolume (15m0s) from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:844 I0505 01:20:58.772500 1 reflector.go:255] Listing and watching *v1.PersistentVolume from sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller/controller.go:844 I0505 01:20:58.872158 1 shared_informer.go:270] caches populated I0505 01:20:58.872202 1 controller.go:859] Started provisioner controller csi-mock-csi-mock-volumes-775_csi-mockplugin-0_b5d74148-b736-4055-802f-d71214094e4d! I0505 01:21:01.278938 1 controller.go:1279] provision "csi-mock-volumes-775/pvc-tk8nd" class "csi-mock-volumes-775-scn9xw8": started I0505 01:21:01.279170 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-775", Name:"pvc-tk8nd", UID:"9f53ef38-e2f7-44f2-b267-9a4513df0b80", APIVersion:"v1", ResourceVersion:"5098281", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "csi-mock-volumes-775/pvc-tk8nd" I0505 01:21:01.279332 1 connection.go:183] GRPC call: /csi.v1.Controller/CreateVolume I0505 01:21:01.279354 1 connection.go:184] GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"topology.hostpath.csi/node":"v125-worker"}},{"segments":{"topology.hostpath.csi/node":"v125-worker2"}}],"requisite":[{"segments":{"topology.hostpath.csi/node":"v125-worker"}},{"segments":{"topology.hostpath.csi/node":"v125-worker2"}}]},"capacity_range":{"required_bytes":1073741824},"name":"pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]} I0505 01:21:01.284141 1 connection.go:186] GRPC response: {"volume":{"accessible_topology":[{"segments":{"topology.hostpath.csi/node":"v125-worker2"}}],"capacity_bytes":1073741824,"volume_id":"18cb8c39-eae3-11ed-a711-86e58b79b215"}} I0505 01:21:01.284362 1 connection.go:187] GRPC error: I0505 01:21:01.284391 1 controller.go:777] create volume rep: {CapacityBytes:1073741824 VolumeId:18cb8c39-eae3-11ed-a711-86e58b79b215 VolumeContext:map[] ContentSource: AccessibleTopology:[segments: ] XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0} I0505 01:21:01.284484 1 controller.go:861] successfully created PV pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 for PVC pvc-tk8nd and csi volume name 18cb8c39-eae3-11ed-a711-86e58b79b215 I0505 01:21:01.284503 1 controller.go:877] successfully created PV {GCEPersistentDisk:nil AWSElasticBlockStore:nil HostPath:nil Glusterfs:nil NFS:nil RBD:nil ISCSI:nil Cinder:nil CephFS:nil FC:nil Flocker:nil FlexVolume:nil AzureFile:nil VsphereVolume:nil Quobyte:nil AzureDisk:nil PhotonPersistentDisk:nil PortworxVolume:nil ScaleIO:nil Local:nil StorageOS:nil CSI:&CSIPersistentVolumeSource{Driver:csi-mock-csi-mock-volumes-775,VolumeHandle:18cb8c39-eae3-11ed-a711-86e58b79b215,ReadOnly:false,FSType:ext4,VolumeAttributes:map[string]string{storage.kubernetes.io/csiProvisionerIdentity: 1683249658669-8081-csi-mock-csi-mock-volumes-775,},ControllerPublishSecretRef:nil,NodeStageSecretRef:nil,NodePublishSecretRef:nil,ControllerExpandSecretRef:nil,}} I0505 01:21:01.284741 1 controller.go:1384] provision "csi-mock-volumes-775/pvc-tk8nd" class "csi-mock-volumes-775-scn9xw8": volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80" provisioned I0505 01:21:01.284765 1 controller.go:1397] provision "csi-mock-volumes-775/pvc-tk8nd" class "csi-mock-volumes-775-scn9xw8": succeeded I0505 01:21:01.284777 1 volume_store.go:154] Saving volume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 I0505 01:21:01.295071 1 volume_store.go:157] Volume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 saved I0505 01:21:01.295149 1 controller.go:1068] Claim processing succeeded, removing PVC 9f53ef38-e2f7-44f2-b267-9a4513df0b80 from claims in progress I0505 01:21:01.295226 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"csi-mock-volumes-775", Name:"pvc-tk8nd", UID:"9f53ef38-e2f7-44f2-b267-9a4513df0b80", APIVersion:"v1", ResourceVersion:"5098281", FieldPath:""}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 I0505 01:21:17.359771 1 controller.go:1413] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": started E0505 01:21:17.359914 1 controller.go:1423] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": volume deletion failed: persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 W0505 01:21:17.360000 1 controller.go:988] Retrying syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", failure 0 E0505 01:21:17.360087 1 controller.go:1006] error syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:17.360129 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", UID:"8ee54324-9abf-44f3-b345-bc8b35e98d9c", APIVersion:"v1", ResourceVersion:"5098472", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:18.360220 1 controller.go:1413] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": started E0505 01:21:18.360389 1 controller.go:1423] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": volume deletion failed: persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 W0505 01:21:18.360455 1 controller.go:988] Retrying syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", failure 1 E0505 01:21:18.360491 1 controller.go:1006] error syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:18.360547 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", UID:"8ee54324-9abf-44f3-b345-bc8b35e98d9c", APIVersion:"v1", ResourceVersion:"5098472", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:20.361672 1 controller.go:1413] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": started E0505 01:21:20.361841 1 controller.go:1423] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": volume deletion failed: persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 W0505 01:21:20.361898 1 controller.go:988] Retrying syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", failure 2 E0505 01:21:20.361933 1 controller.go:1006] error syncing volume "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:20.362006 1 event.go:282] Event(v1.ObjectReference{Kind:"PersistentVolume", Namespace:"", Name:"pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80", UID:"8ee54324-9abf-44f3-b345-bc8b35e98d9c", APIVersion:"v1", ResourceVersion:"5098472", FieldPath:""}): type: 'Warning' reason: 'VolumeFailedDelete' persistentvolume pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80 is still attached to node v125-worker2 I0505 01:21:24.362112 1 controller.go:1413] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": started I0505 01:21:24.362248 1 connection.go:183] GRPC call: /csi.v1.Controller/DeleteVolume I0505 01:21:24.362269 1 connection.go:184] GRPC request: {"volume_id":"18cb8c39-eae3-11ed-a711-86e58b79b215"} I0505 01:21:24.364899 1 connection.go:186] GRPC response: {} I0505 01:21:24.364993 1 connection.go:187] GRPC error: I0505 01:21:24.365011 1 controller.go:1428] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": volume deleted I0505 01:21:24.369903 1 controller.go:1478] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": persistentvolume deleted I0505 01:21:24.369939 1 controller.go:1483] delete "pvc-9f53ef38-e2f7-44f2-b267-9a4513df0b80": succeeded ==== end of pod log for container csi-mockplugin-0/csi-provisioner ====