{"id":40567,"date":"2023-09-29T02:22:31","date_gmt":"2023-09-29T02:22:31","guid":{"rendered":"https:\/\/www.devopsschool.com\/blog\/?p=40567"},"modified":"2025-01-23T12:45:12","modified_gmt":"2025-01-23T12:45:12","slug":"troubleshooting-kubernetes-networking-with-calico","status":"publish","type":"post","link":"https:\/\/www.devopsschool.com\/blog\/troubleshooting-kubernetes-networking-with-calico\/","title":{"rendered":"Troubleshooting Kubernetes Networking with Tigera Calico"},"content":{"rendered":"\n<p><a href=\"https:\/\/docs.tigera.io\/v3.2\/maintenance\/troubleshoot\/troubleshooting\" target=\"_blank\" rel=\"noopener\">Tigera Calico Troubleshooting and diagnostics<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/docs.tigera.io\/v3.2\/maintenance\/troubleshoot\/component-logs\" target=\"_blank\" rel=\"noopener\">Tigera Calico Component logs<\/a><\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n<pre class=\"wp-block-code\" aria-describedby=\"shcb-language-1\" data-shcb-language-name=\"PHP\" data-shcb-language-slug=\"php\"><span><code class=\"hljs language-php\"><span class=\"hljs-comment\">#!\/bin\/bash<\/span>\r\n\r\n<span class=\"hljs-keyword\">if<\/span> &#91;&#91; <span class=\"hljs-string\">\"$1\"<\/span> != <span class=\"hljs-string\">\"diags\"<\/span> ]]\r\nthen\r\n    <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Unsupported command: $1\"<\/span>\r\n    <span class=\"hljs-keyword\">exit<\/span> <span class=\"hljs-number\">1<\/span>\r\nfi\r\nshift <span class=\"hljs-number\">1<\/span>\r\n\r\nSINCE=<span class=\"hljs-number\">0<\/span>s  <span class=\"hljs-comment\"># Only return logs newer than relative duration such as 5s, 2m, or 3h. Defaults to all logs.<\/span>\r\n\r\n<span class=\"hljs-comment\"># Parse optional flag(s):<\/span>\r\n<span class=\"hljs-comment\">#   --since=relative_duration (e.g. 10s, 5m)<\/span>\r\n\r\n<span class=\"hljs-keyword\">while<\/span> (( <span class=\"hljs-string\">\"$#\"<\/span> )); <span class=\"hljs-keyword\">do<\/span>\r\n  <span class=\"hljs-keyword\">case<\/span> <span class=\"hljs-string\">\"$1\"<\/span> in\r\n    --since=*)\r\n      str=$<span class=\"hljs-number\">1<\/span>\r\n      SINCE=<span class=\"hljs-string\">\"${str#*=}\"<\/span>    <span class=\"hljs-comment\"># grab everything after '='<\/span>\r\n      shift <span class=\"hljs-number\">1<\/span>\r\n\r\n      <span class=\"hljs-keyword\">if<\/span> &#91;&#91; ! $SINCE =~ ^&#91;<span class=\"hljs-number\">0<\/span><span class=\"hljs-number\">-9<\/span>]+&#91;smh]$ ]]; then\r\n          <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"invalid relative duration, try 10s, 5m, or 1h\"<\/span>\r\n          <span class=\"hljs-keyword\">exit<\/span> <span class=\"hljs-number\">1<\/span>\r\n      fi\r\n      ;;\r\n    -*|--*=) <span class=\"hljs-comment\"># unsupported flags<\/span>\r\n      <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Error: Unsupported flag $1\"<\/span> &gt;&amp;<span class=\"hljs-number\">2<\/span>\r\n      <span class=\"hljs-keyword\">exit<\/span> <span class=\"hljs-number\">1<\/span>\r\n      ;;\r\n    *) <span class=\"hljs-comment\"># unsupported argument<\/span>\r\n      <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Error: Unsupported argument $1\"<\/span> &gt;&amp;<span class=\"hljs-number\">2<\/span>\r\n      <span class=\"hljs-keyword\">exit<\/span> <span class=\"hljs-number\">1<\/span>\r\n      ;;\r\n  esac\r\ndone\r\n\r\n<span class=\"hljs-comment\"># Check pre-requisites, like a functioning kubectl.<\/span>\r\n<span class=\"hljs-keyword\">if<\/span> &#91; <span class=\"hljs-string\">\"$(which kubectl)\"<\/span> == <span class=\"hljs-string\">\"\"<\/span> ]; then <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Unable to locate kubectl in PATH\"<\/span>; fi\r\nkubectl get ns <span class=\"hljs-number\">2<\/span>&gt;&amp;<span class=\"hljs-number\">1<\/span>&gt;\/dev\/<span class=\"hljs-keyword\">null<\/span> || <span class=\"hljs-keyword\">if<\/span> &#91;&#91; $? != <span class=\"hljs-number\">0<\/span> ]]; then <span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"kubectl does not appear to be functioning\"<\/span>; <span class=\"hljs-keyword\">exit<\/span> <span class=\"hljs-number\">1<\/span>; fi\r\n\r\n<span class=\"hljs-comment\"># Make a tmp dir.<\/span>\r\nset -e\r\nmtmp=$(mktemp -d)\r\nmkdir ${mtmp}\/calico-diagnostics\r\ntmp=${mtmp}\/calico-diagnostics\r\nset +e\r\n\r\n\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"==== Begin collecting diagnostics. ====\"<\/span>\r\n\r\n<span class=\"hljs-comment\"># Get some basic cluster state.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting basic cluster state...\"<\/span>\r\nkubectl get ns &gt; ${tmp}\/namespaces.txt\r\nkubectl get all -n calico-system -o wide &gt; ${tmp}\/calico-system.txt\r\nkubectl get all -n tigera-operator -o wide &gt; ${tmp}\/tigera-operator.txt\r\n\r\nmkdir ${tmp}\/operator.tigera.io\r\nkubectl get installations -o yaml &gt; ${tmp}\/operator.tigera.io\/installations.yaml\r\nkubectl get apiservers -o yaml &gt; ${tmp}\/operator.tigera.io\/apiservers.yaml\r\nkubectl get compliances -o yaml &gt; ${tmp}\/operator.tigera.io\/compliances.yaml\r\nkubectl get intrusiondetections -o yaml &gt; ${tmp}\/operator.tigera.io\/intrusiondetections.yaml\r\nkubectl get managers -o yaml &gt; ${tmp}\/operator.tigera.io\/managers.yaml\r\nkubectl get logcollectors -o yaml &gt; ${tmp}\/operator.tigera.io\/logcollectors.yaml\r\nkubectl get logstorages -o yaml &gt; ${tmp}\/operator.tigera.io\/logstorages.yaml\r\nkubectl get managementclusterconnections -o yaml &gt; ${tmp}\/operator.tigera.io\/managementclusterconnections.yaml\r\n\r\n<span class=\"hljs-comment\"># Get tigera status.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting TigeraStatus details...\"<\/span>\r\nkubectl get tigerastatus &gt; ${tmp}\/tigerastatus.txt\r\nkubectl get tigerastatus -o yaml &gt; ${tmp}\/tigerastatus-yaml.txt\r\n\r\n<span class=\"hljs-comment\"># Get nodes.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting Node details...\"<\/span>\r\nkubectl get nodes -o wide &gt; ${tmp}\/nodes.txt\r\nkubectl get nodes -o yaml &gt; ${tmp}\/nodes-yaml.txt\r\n\r\n<span class=\"hljs-comment\"># Get IPAM information.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting IPAM diagnostics...\"<\/span>\r\nmkdir -p ${tmp}\/ipam\r\nkubectl get ipamblocks -o yaml &gt; ${tmp}\/ipam\/ipamblocks.txt\r\nkubectl get blockaffinities -o yaml &gt; ${tmp}\/ipam\/blockaffinities.txt\r\nkubectl get ipamhandles -o yaml &gt; ${tmp}\/ipam\/ipamhandles.txt\r\n\r\n<span class=\"hljs-comment\"># Get operator logs. <\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting tigera-operator logs...\"<\/span>\r\nkubectl logs --since=$SINCE -n tigera-operator -l k8s-app=tigera-operator &gt; ${tmp}\/tigera-operator.logs\r\n\r\n<span class=\"hljs-comment\"># Get typha logs.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting calico\/typha logs...\"<\/span>\r\nmkdir -p ${tmp}\/typhas\r\n<span class=\"hljs-keyword\">for<\/span> typha in $(kubectl get pods -n calico-system -l k8s-app=calico-typha -o go-template --template=<span class=\"hljs-string\">\"{{range .items}}{{.metadata.name}} {{end}}\"<\/span>); <span class=\"hljs-keyword\">do<\/span>\r\n\tkubectl logs --since=$SINCE -n calico-system $typha &gt; ${tmp}\/typhas\/${typha}.log\r\ndone\r\n\r\n<span class=\"hljs-comment\"># Get per-node logs and network information.<\/span>\r\nmkdir -p ${tmp}\/nodes\r\n<span class=\"hljs-keyword\">for<\/span> node in $(kubectl get pods -n calico-system -l k8s-app=calico-node -o go-template --template=<span class=\"hljs-string\">\"{{range .items}}{{.metadata.name}} {{end}}\"<\/span>); <span class=\"hljs-keyword\">do<\/span>\r\n\t<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Collecting logs for node: $node\"<\/span>\r\n\tmkdir -p ${tmp}\/nodes\/${node}\r\n\tkubectl logs --since=$SINCE -n calico-system $node &gt; ${tmp}\/nodes\/${node}\/${node}.log\r\n\tkubectl exec -n calico-system -t $node -- iptables-save -c &gt; ${tmp}\/nodes\/${node}\/iptables-save.txt\r\n\tkubectl exec -n calico-system -t $node -- ip route &gt; ${tmp}\/nodes\/${node}\/iproute.txt\r\ndone\r\n\r\n<span class=\"hljs-comment\"># Tar it all up for easy sharing.<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"\"<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"==== Producing a diagnostics bundle. ====\"<\/span>\r\nrm -f calico-diagnostics.tar.gz\r\ntar cfz ${mtmp}\/calico-diagnostics.tar.gz -C ${mtmp} calico-diagnostics\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"\"<\/span>\r\n<span class=\"hljs-keyword\">echo<\/span> <span class=\"hljs-string\">\"Diagnostic bundle produced at ${mtmp}\/calico-diagnostics.tar.gz\"<\/span><\/code><\/span><small class=\"shcb-language\" id=\"shcb-language-1\"><span class=\"shcb-language__label\">Code language:<\/span> <span class=\"shcb-language__name\">PHP<\/span> <span class=\"shcb-language__paren\">(<\/span><span class=\"shcb-language__slug\">php<\/span><span class=\"shcb-language__paren\">)<\/span><\/small><\/pre>\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<script src=\"https:\/\/gist.github.com\/devops-school\/54352fe26393e1202efd102c520cf893.js\"><\/script>\n","protected":false},"excerpt":{"rendered":"<p>Tigera Calico Troubleshooting and diagnostics Tigera Calico Component logs<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_joinchat":[],"footnotes":""},"categories":[4859],"tags":[],"class_list":["post-40567","post","type-post","status-publish","format-standard","hentry","category-kubernetes"],"_links":{"self":[{"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/posts\/40567","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/comments?post=40567"}],"version-history":[{"count":4,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/posts\/40567\/revisions"}],"predecessor-version":[{"id":48273,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/posts\/40567\/revisions\/48273"}],"wp:attachment":[{"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/media?parent=40567"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/categories?post=40567"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.devopsschool.com\/blog\/wp-json\/wp\/v2\/tags?post=40567"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}