==== start of pod log for container csi-mockplugin-attacher-0/csi-attacher ====
I0203 01:05:39.429496       1 main.go:99] Version: v3.3.0
I0203 01:05:39.451537       1 connection.go:154] Connecting to unix:///csi/csi.sock
I0203 01:05:40.459284       1 common.go:111] Probing CSI driver for readiness
I0203 01:05:40.459307       1 connection.go:183] GRPC call: /csi.v1.Identity/Probe
I0203 01:05:40.459313       1 connection.go:184] GRPC request: {}
I0203 01:05:40.470011       1 connection.go:186] GRPC response: {}
I0203 01:05:40.470222       1 connection.go:187] GRPC error: <nil>
I0203 01:05:40.470247       1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo
I0203 01:05:40.470252       1 connection.go:184] GRPC request: {}
I0203 01:05:40.471529       1 connection.go:186] GRPC response: {"name":"csi-mock-csi-mock-volumes-6839","vendor_version":"v1.7.3"}
I0203 01:05:40.471659       1 connection.go:187] GRPC error: <nil>
I0203 01:05:40.471680       1 main.go:155] CSI driver name: "csi-mock-csi-mock-volumes-6839"
I0203 01:05:40.471694       1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginCapabilities
I0203 01:05:40.471702       1 connection.go:184] GRPC request: {}
I0203 01:05:40.477163       1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Service":{"type":1}}},{"Type":{"Service":{"type":2}}}]}
I0203 01:05:40.477309       1 connection.go:187] GRPC error: <nil>
I0203 01:05:40.477318       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities
I0203 01:05:40.477323       1 connection.go:184] GRPC request: {}
I0203 01:05:40.478932       1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":12}}},{"Type":{"Rpc":{"type":4}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":6}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":11}}},{"Type":{"Rpc":{"type":9}}},{"Type":{"Rpc":{"type":2}}}]}
I0203 01:05:40.479267       1 connection.go:187] GRPC error: <nil>
I0203 01:05:40.480908       1 main.go:230] CSI driver supports ControllerPublishUnpublish, using real CSI handler
I0203 01:05:40.482710       1 controller.go:128] Starting CSI attacher
I0203 01:05:40.484442       1 reflector.go:219] Starting reflector *v1.VolumeAttachment (10m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.484477       1 reflector.go:255] Listing and watching *v1.VolumeAttachment from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.484507       1 reflector.go:219] Starting reflector *v1.CSINode (10m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.484513       1 reflector.go:219] Starting reflector *v1.PersistentVolume (10m0s) from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.484539       1 reflector.go:255] Listing and watching *v1.PersistentVolume from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.484530       1 reflector.go:255] Listing and watching *v1.CSINode from k8s.io/client-go/informers/factory.go:134
I0203 01:05:40.582929       1 shared_informer.go:270] caches populated
I0203 01:05:44.343532       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.343571       1 csi_handler.go:221] CSIHandler: processing VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.343598       1 csi_handler.go:248] Attaching "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.343607       1 csi_handler.go:427] Starting attach operation for "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.343677       1 csi_handler.go:347] Adding finalizer to PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:05:44.350611       1 csi_handler.go:356] PV finalizer added to "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:05:44.350656       1 csi_handler.go:746] Found NodeID v125-worker2 in CSINode v125-worker2
I0203 01:05:44.350739       1 csi_handler.go:309] VA finalizer added to "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.350756       1 csi_handler.go:323] NodeID annotation added to "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.354672       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerPublishVolume
I0203 01:05:44.354716       1 connection.go:184] GRPC request: {"node_id":"v125-worker2","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}},"volume_context":{"storage.kubernetes.io/csiProvisionerIdentity":"1675386340649-8081-csi-mock-csi-mock-volumes-6839"},"volume_id":"e1c6e109-a35e-11ed-a294-66eb4bca1485"}
I0203 01:05:44.356604       1 connection.go:186] GRPC response: {}
I0203 01:05:44.356729       1 connection.go:187] GRPC error: <nil>
I0203 01:05:44.356744       1 csi_handler.go:261] Attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.356776       1 util.go:38] Marking as attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360737       1 util.go:52] Marked as attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360768       1 csi_handler.go:267] Fully attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360784       1 csi_handler.go:237] CSIHandler: finished processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360831       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360849       1 csi_handler.go:221] CSIHandler: processing VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360861       1 csi_handler.go:248] Attaching "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360874       1 csi_handler.go:427] Starting attach operation for "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360908       1 csi_handler.go:341] PV finalizer is already set on "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:05:44.360932       1 csi_handler.go:746] Found NodeID v125-worker2 in CSINode v125-worker2
I0203 01:05:44.360953       1 csi_handler.go:301] VA finalizer is already set on "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360969       1 csi_handler.go:315] NodeID annotation is already set on "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.360988       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerPublishVolume
I0203 01:05:44.361002       1 connection.go:184] GRPC request: {"node_id":"v125-worker2","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}},"volume_context":{"storage.kubernetes.io/csiProvisionerIdentity":"1675386340649-8081-csi-mock-csi-mock-volumes-6839"},"volume_id":"e1c6e109-a35e-11ed-a294-66eb4bca1485"}
I0203 01:05:44.361855       1 connection.go:186] GRPC response: {}
I0203 01:05:44.361911       1 connection.go:187] GRPC error: <nil>
I0203 01:05:44.361930       1 csi_handler.go:261] Attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.361955       1 util.go:38] Marking as attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365318       1 util.go:52] Marked as attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365336       1 csi_handler.go:267] Fully attached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365347       1 csi_handler.go:237] CSIHandler: finished processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365379       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365391       1 csi_handler.go:221] CSIHandler: processing VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:44.365400       1 csi_handler.go:243] "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00" is already attached
I0203 01:05:44.365409       1 csi_handler.go:237] CSIHandler: finished processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:05:49.377981       1 controller.go:208] Started VA processing "csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64"
I0203 01:05:49.378023       1 controller.go:223] Skipping VolumeAttachment csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64 for attacher csi-mock-csi-mock-volumes-6953
I0203 01:05:49.391480       1 controller.go:208] Started VA processing "csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64"
I0203 01:05:49.391502       1 controller.go:223] Skipping VolumeAttachment csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64 for attacher csi-mock-csi-mock-volumes-6953
I0203 01:05:49.398454       1 controller.go:208] Started VA processing "csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64"
I0203 01:05:49.398477       1 controller.go:223] Skipping VolumeAttachment csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64 for attacher csi-mock-csi-mock-volumes-6953
I0203 01:06:04.986934       1 controller.go:208] Started VA processing "csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f"
I0203 01:06:04.986997       1 controller.go:223] Skipping VolumeAttachment csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f for attacher csi-mock-csi-mock-volumes-3275
I0203 01:06:05.003821       1 controller.go:208] Started VA processing "csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f"
I0203 01:06:05.003853       1 controller.go:223] Skipping VolumeAttachment csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f for attacher csi-mock-csi-mock-volumes-3275
I0203 01:06:05.010795       1 controller.go:208] Started VA processing "csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f"
I0203 01:06:05.010845       1 controller.go:223] Skipping VolumeAttachment csi-899b2aeee189bad76b3d53e8e7fdd45d030051908b0cf203d03d7720308d307f for attacher csi-mock-csi-mock-volumes-3275
I0203 01:06:07.929446       1 controller.go:208] Started VA processing "csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64"
I0203 01:06:07.929489       1 controller.go:223] Skipping VolumeAttachment csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64 for attacher csi-mock-csi-mock-volumes-6953
I0203 01:06:07.936759       1 controller.go:208] Started VA processing "csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64"
I0203 01:06:07.936797       1 controller.go:223] Skipping VolumeAttachment csi-01629fe05a7d36c64ffd5894f11998746fa274dbc2f2b93858b9c974832e2a64 for attacher csi-mock-csi-mock-volumes-6953
I0203 01:06:07.943642       1 controller.go:259] Started PV processing "pvc-aebf4305-695b-4e1a-9052-2258f6dd568f"
I0203 01:06:07.943684       1 csi_handler.go:629] CSIHandler: processing PV "pvc-aebf4305-695b-4e1a-9052-2258f6dd568f"
I0203 01:06:07.943724       1 csi_handler.go:651] CSIHandler: processing PV "pvc-aebf4305-695b-4e1a-9052-2258f6dd568f": no deletion timestamp, ignoring
I0203 01:06:12.673119       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.673188       1 csi_handler.go:221] CSIHandler: processing VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.673215       1 csi_handler.go:272] Starting detach operation for "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.673327       1 csi_handler.go:279] Detaching "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.673370       1 csi_handler.go:746] Found NodeID v125-worker2 in CSINode v125-worker2
I0203 01:06:12.673408       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerUnpublishVolume
I0203 01:06:12.673428       1 connection.go:184] GRPC request: {"node_id":"v125-worker2","volume_id":"e1c6e109-a35e-11ed-a294-66eb4bca1485"}
I0203 01:06:12.675278       1 connection.go:186] GRPC response: {}
I0203 01:06:12.675425       1 connection.go:187] GRPC error: <nil>
I0203 01:06:12.675447       1 csi_handler.go:587] Detached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.675505       1 util.go:80] Marking as detached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.688890       1 util.go:106] Finalizer removed from "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.688931       1 csi_handler.go:292] Fully detached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.688948       1 csi_handler.go:237] CSIHandler: finished processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.689005       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.689027       1 csi_handler.go:221] CSIHandler: processing VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.689038       1 csi_handler.go:272] Starting detach operation for "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.689074       1 csi_handler.go:279] Detaching "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.689101       1 csi_handler.go:746] Found NodeID v125-worker2 in CSINode v125-worker2
I0203 01:06:12.689118       1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerUnpublishVolume
I0203 01:06:12.689139       1 connection.go:184] GRPC request: {"node_id":"v125-worker2","volume_id":"e1c6e109-a35e-11ed-a294-66eb4bca1485"}
I0203 01:06:12.689110       1 controller.go:259] Started PV processing "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:12.689298       1 csi_handler.go:629] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:12.689346       1 csi_handler.go:651] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8": no deletion timestamp, ignoring
I0203 01:06:12.690003       1 connection.go:186] GRPC response: {}
I0203 01:06:12.690049       1 connection.go:187] GRPC error: <nil>
I0203 01:06:12.690061       1 csi_handler.go:587] Detached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.690098       1 util.go:80] Marking as detached "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.694099       1 csi_handler.go:613] Saving detach error to "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:12.697455       1 csi_handler.go:286] Failed to save detach error to "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00": volumeattachments.storage.k8s.io "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00" not found
I0203 01:06:12.697492       1 csi_handler.go:231] Error processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00": failed to detach: could not mark as detached: volumeattachments.storage.k8s.io "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00" not found
I0203 01:06:13.697987       1 controller.go:208] Started VA processing "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00"
I0203 01:06:13.698104       1 controller.go:215] VA "csi-121adb451d0a7fdd249c30305d1a373e02ead4ce86ea604adb8bb55f20491f00" deleted, ignoring
I0203 01:06:15.175389       1 controller.go:259] Started PV processing "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:15.175436       1 csi_handler.go:629] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:15.175541       1 csi_handler.go:690] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8": no VA found, removing finalizer
E0203 01:06:15.183857       1 csi_handler.go:707] Failed to remove finalizer from PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8": PersistentVolume "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8" is invalid: metadata.finalizers: Forbidden: no new finalizers can be added if the object is being deleted, found new finalizers []string{"kubernetes.io/pv-protection"}
I0203 01:06:15.183953       1 controller.go:259] Started PV processing "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:15.183986       1 csi_handler.go:629] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:15.184018       1 csi_handler.go:690] CSIHandler: processing PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8": no VA found, removing finalizer
I0203 01:06:15.191772       1 csi_handler.go:712] Removed finalizer from PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:16.184207       1 controller.go:259] Started PV processing "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8"
I0203 01:06:16.184277       1 controller.go:266] PV "pvc-3444e7f3-6c41-4752-9026-8db0f99813f8" deleted, ignoring
==== end of pod log for container csi-mockplugin-attacher-0/csi-attacher ====