Rotate Kubernetes Certificates
During SevOne Data Insight upgrade, the k3s service automatically rotates certificates that are due to expire within 90 days. In the event that they expire before k3s is able to rotate them, you will need to rotate manually.
$ kubectl get pods
Unable to connect to the server: x509: certificate has expired or is not yet valid
Backup TLS Directory
As a precautionary measure, backup the TLS directory.
$ sudo tar -czvf /var/lib/rancher/k3s/server/tls.tgz /var/lib/rancher/k3s/server/tlsGenerate New Certificates
- Remove the cached certificate from a Kubernetes secret.
$ sudo rm /var/lib/rancher/k3s/server/tls/dynamic-cert.json - Restart k3s service to rotate the certificates.
$ sudo systemctl restart k3sImportant: for Multi-Node environment, verify if the server certificates have been rotated by executing the following commands from the control-plane node.$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/server/tls/*.crt`; do echo $i; \ openssl x509 -enddate -noout -in $i; echo "---"; done' -b serveruserXYZ-di | CHANGED | rc=0 >> /var/lib/rancher/k3s/server/tls/client-admin.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-auth-proxy.crt notAfter=Aug 18 14:20:19 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/client-ca.nochain.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/client-controller.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-k3s-cloud-controller.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-scheduler.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-supervisor.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/request-header-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/server-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/server-ca.nochain.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt notAfter=Aug 18 14:20:19 2026 GMT ---
$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/agent/*.crt`; do echo $i; \ openssl x509 -enddate -noout -in $i; echo "---"; done' -b serveruserXYZ-di | CHANGED | rc=0 >> /var/lib/rancher/k3s/agent/client-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/client-k3s-controller.crt notAfter=Aug 18 14:20:20 2026 GMT --- /var/lib/rancher/k3s/agent/client-kubelet.crt notAfter=Aug 18 14:20:20 2026 GMT --- /var/lib/rancher/k3s/agent/client-kube-proxy.crt notAfter=Aug 18 14:20:20 2026 GMT --- /var/lib/rancher/k3s/agent/server-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/serving-kubelet.crt notAfter=Aug 18 14:20:20 2026 GMT ---
Restart k3s-agent; the following is an example with one agent.$ ansible -m systemd -a 'state=restarted name=k3s-agent' -b agentuserXYZ-di1 | CHANGED => { "ansible_facts": { "discovered_interpreter_python": "/usr/libexec/platform-python" }, "changed": true, "name": "k3s-agent", "state": "started", "status": { "ActiveEnterTimestamp": "Mon 2025-08-18 14:51:32 UTC", "ActiveEnterTimestampMonotonic": "312666985333", "ActiveExitTimestamp": "Mon 2025-08-18 14:51:16 UTC", "ActiveExitTimestampMonotonic": "312650654557", "ActiveState": "active", "After": "basic.target sysinit.target systemd-journald.socket system.slice network-online.target", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "yes", "AssertTimestamp": "Mon 2025-08-18 14:51:16 UTC", "AssertTimestampMonotonic": "312650701452", "Before": "multi-user.target shutdown.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "no", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", "CollectMode": "inactive", "ConditionResult": "yes", "ConditionTimestamp": "Mon 2025-08-18 14:51:16 UTC", "ConditionTimestampMonotonic": "312650701452", "ConfigurationDirectoryMode": "0755", "Conflicts": "shutdown.target", "ControlGroup": "/system.slice/k3s-agent.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "yes", "DelegateControllers": "cpu cpuacct cpuset io blkio memory devices pids", "Description": "Lightweight Kubernetes", "DevicePolicy": "auto", "Documentation": "https://k3s.io", "DropInPaths": "/etc/systemd/system/k3s-agent.service.d/override.conf", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "Environment": "CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS=1", "EnvironmentFiles": "/etc/systemd/system/k3s-agent.service.env (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "3327895", "ExecMainStartTimestamp": "Mon 2025-08-18 14:51:16 UTC", "ExecMainStartTimestampMonotonic": "312650751028", "ExecMainStatus": "0", "ExecStart": "{ path=/usr/local/bin/k3s ; argv[]=/usr/local/bin/k3s agent ; ignore_errors=no ; start_time=[Mon 2025-08-18 14:51:16 UTC] ; stop_time=[n/a] ; pid=3327895 ; code=(null) ; status=0/0 }", "ExecStartPre": "{ path=/sbin/modprobe ; argv[]=/sbin/modprobe overlay ; ignore_errors=yes ; start_time=[Mon 2025-08-18 14:51:16 UTC] ; stop_time=[Mon 2025-08-18 14:51:16 UTC] ; pid=3327890 ; code=exited ; status=0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/etc/systemd/system/k3s-agent.service", "FreezerState": "running", "GID": "[not set]", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "k3s-agent.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestamp": "Mon 2025-08-18 14:51:16 UTC", "InactiveEnterTimestampMonotonic": "312650701029", "InactiveExitTimestamp": "Mon 2025-08-18 14:51:16 UTC", "InactiveExitTimestampMonotonic": "312650703600", "InvocationID": "861af65dfad64d21bf0eba0869cc7ece", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "process", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinity", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": "0", "LimitNICESoft": "0", "LimitNOFILE": "1048576", "LimitNOFILESoft": "1048576", "LimitNPROC": "infinity", "LimitNPROCSoft": "infinity", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "124372", "LimitSIGPENDINGSoft": "124372", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "3327895", "MemoryAccounting": "yes", "MemoryCurrent": "774283264", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "k3s-agent.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "main", "OOMScoreAdjust": "0", "OnFailureJobMode": "replace", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "no", "PrivateUsers": "no", "ProtectControlGroups": "no", "ProtectHome": "no", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "no", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "system.slice sysinit.target", "Restart": "always", "RestartUSec": "5s", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "inherit", "StandardInput": "null", "StandardInputData": "", "StandardOutput": "journal", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestamp": "Mon 2025-08-18 14:51:32 UTC", "StateChangeTimestampMonotonic": "312666985333", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "running", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "400", "TasksMax": "infinity", "TimeoutStartUSec": "infinity", "TimeoutStopUSec": "1min 30s", "TimerSlackNSec": "50000", "Transient": "no", "Type": "notify", "UID": "[not set]", "UMask": "0022", "UnitFilePreset": "disabled", "UnitFileState": "enabled", "UtmpMode": "init", "WantedBy": "multi-user.target", "Wants": "network-online.target", "WatchdogTimestamp": "Mon 2025-08-18 14:51:32 UTC", "WatchdogTimestampMonotonic": "312666985330", "WatchdogUSec": "0" } }
Refresh Kubernetes Config
After rotating the Kubernetes certificates, the Kubernetes configuration file must be refreshed to apply the new certificates.
Refresh Kubernetes config file
for 'root' user
$ sudo cp /etc/rancher/k3s/k3s.yaml /root/.kube/config
for 'sevone' user
$ sudo cp /etc/rancher/k3s/k3s.yaml /home/sevone/.kube/config
$ sudo chown -R sevone:sevone /home/sevone/.kube
Note: You can now run Kubernetes commands. This will allow you to backup your all-important security keys in case you have not done so already.
Verify Certificates
To verify the certificates, execute the following commands.
$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/server/tls/*.crt`; do echo $i; \
openssl x509 -enddate -noout -in $i; echo "---"; done' -b server
$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/agent/*.crt`; do echo $i; \
openssl x509 -enddate -noout -in $i; echo "---"; done' -b all
Example# 1: Certificates about to expire on the server
$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/server/tls/*.crt`; do echo $i; \
openssl x509 -enddate -noout -in $i; echo "---"; done' -b server
userXYZ-di | CHANGED | rc=0 >> /var/lib/rancher/k3s/server/tls/client-admin.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-auth-proxy.crt notAfter=Aug 18 14:20:19 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/client-ca.nochain.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/client-controller.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-k3s-cloud-controller.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-scheduler.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/client-supervisor.crt notAfter=Aug 18 14:20:18 2026 GMT --- /var/lib/rancher/k3s/server/tls/request-header-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/server-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/server-ca.nochain.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt notAfter=Aug 18 14:20:19 2026 GMT ---
Example# 2: Certificates about to expire across the cluster
$ ansible -m shell -a 'for i in `ls /var/lib/rancher/k3s/agent/*.crt`; do echo $i; \
openssl x509 -enddate -noout -in $i; echo "---"; done' -b all
userXYZ-di | CHANGED | rc=0 >> /var/lib/rancher/k3s/agent/client-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/client-k3s-controller.crt notAfter=Aug 18 14:20:57 2026 GMT --- /var/lib/rancher/k3s/agent/client-kubelet.crt notAfter=Aug 18 14:20:57 2026 GMT --- /var/lib/rancher/k3s/agent/client-kube-proxy.crt notAfter=Aug 18 14:20:57 2026 GMT --- /var/lib/rancher/k3s/agent/server-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/serving-kubelet.crt notAfter=Aug 18 14:20:56 2026 GMT --- userXYZ-di1 | CHANGED | rc=0 >> /var/lib/rancher/k3s/agent/client-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/client-k3s-controller.crt notAfter=Aug 18 14:51:17 2026 GMT --- /var/lib/rancher/k3s/agent/client-kubelet.crt notAfter=Aug 18 14:51:17 2026 GMT --- /var/lib/rancher/k3s/agent/client-kube-proxy.crt notAfter=Aug 18 14:51:17 2026 GMT --- /var/lib/rancher/k3s/agent/server-ca.crt notAfter=Aug 11 19:38:58 2035 GMT --- /var/lib/rancher/k3s/agent/serving-kubelet.crt notAfter=Aug 18 14:51:17 2026 GMT ---