feat: add system-upgrade controller and plan k3s plan

k8s/argo-apps/loki.yaml

@@ -20,7 +20,6 @@ spec:
       helm:
         valuesObject:
           grafana:
-            replicas: 1
             enabled: true
             persistence:
               type: pvc
@@ -57,13 +56,16 @@ spec:
                 - host: loki.fuku
                   paths:
                     - /
-
           prometheus:
             enabled: true
             isDefault: true
             server:
               persistentVolume:
                 enabled: true
+            extraScrapeConfigs: |
+              - job_name: 'argocd'
+                static_configs: 
+                  - targets: ["argocd-server-metrics.argocd.svc.cluster.local:8083"]
             alertmanager:
               persistence:
                 enabled: true

k8s/argo-apps/system-upgrade-controller.yml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: system-upgrade
+  namespace: argocd
+spec:
+  destination:
+    name: ''
+    namespace: 'system-upgrade'
+    server: "https://kubernetes.default.svc"
+  sources:
+    - repoURL: https://github.com/rancher/system-upgrade-controller.git
+      targetRevision: v0.13.2
+      path: manifests/
+    - repoURL: https://git.roboces.dev/catalin/fukuops.git
+      path: k8s/services/argo
+      targetRevision: main
+  syncPolicy:
+    automated: { }
+  project: management

k8s/services/argo/project-fuku.yaml

@@ -8,6 +8,8 @@ spec:
   destinations:
     - namespace: apps-fuku
       server: https://kubernetes.default.svc
+    - namespace: postgres
+      server: https://kubernetes.default.svc
   clusterResourceWhitelist:
     - group: rbac.authorization.k8s.io
       kind: ClusterRole
@@ -19,3 +21,5 @@ spec:
     - https://charts.goauthentik.io/
     - https://kubereboot.github.io/charts
     - https://sqljames.github.io/factorio-server-charts/
+    - https://portainer.github.io/k8s/
+    - https://charts.bitnami.com/bitnami

k8s/services/argo/project-management.yaml

@@ -14,6 +14,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: monitoring
       server: https://kubernetes.default.svc
+    - namespace: system-upgrade
+      server: https://kubernetes.default.svc
   clusterResourceWhitelist:
     - group: "*"
       kind: "*"
@@ -24,3 +26,4 @@ spec:
     - https://bitnami-labs.github.io/sealed-secrets
     - https://grafana.github.io/helm-charts
     - https://kubernetes-sigs.github.io/descheduler/
+    - https://github.com/rancher/system-upgrade-controller.git

k8s/services/miniflux/sealedsecrets.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: bitnami.com/v1alpha1
 kind: SealedSecret
 metadata:
@@ -6,11 +7,11 @@ metadata:
   namespace: apps-roboces
 spec:
   encryptedData:
-    ADMIN_PASSWORD: AgCDa9wDtOtrc6jY7T7NLM4zwo5UJTZHPenu5QcxNd3XQWoTBZmm76Btwyn1eGgz9d3IOYYbKb9x8sHIIm0z2Fz17YO7sZsZ1snVLRlKk+Udbdr8vDD/geF4lqeMCR44ie6Y1Kgm8p8MJ63DslHkcgJFpw9Dd09BCLZZn84D/wNdIx9Ya0yDDgi2N4y3F3fNmx1Eh8N4n3C3Y/SKD2LVAzyyzDu0LqbCY5u+etKP96NAXl57nxoJPIaa2iqvIcvLur7KDUT3UmCeWAl0CppIXV/AEW8yHd/cSt/ukWpKIsq5aRmS+gsR3WpnVQrv/miVHdzY4nz/iEtl44W3t2lQvf2TUQmRzi8ubuPtu/JE6rASXZKRAR9LQClL/GX0iU5Y0iHX1z+Fe7M4CPCHSAHkPsy1rZuPa5NB56ywAbQIhiUoKNkYSQBQ5q4LBmaOmPDOW/Mwro2QQV9m1klXs/9xBTi8Zjcpdi+d38meflN3pw5sfcPsAWz3M5c+i+r2uDqZymSqmwUPaHrXAcuQCEeXTw07IJtBPFY3uHQfHo/STCparyKNmYo9Kzm4UAIu+j9GetSk6AoTEAoqd9Xpg2h/DrRL7izwfJQXvQKjqwX3wixlhzaFTD8xn7/+24pVzjjAPsOR43CMn9o8C05FkNNY2b8uL9dZysu4Donop17o4QkgirfNLksoWeEeGaG/NMrOkQ+O7rdG06pC1cEqMI6G2w==
+    ADMIN_PASSWORD: AgAEV8w3iDaOhxZJWo1XmUab9No/T5g6ci92Py75+4ijVfgRCTpHsXT7pRBGWrHzfprfvMKUCNn3rxJ7PbUZBRevLtSFXHXP35s1f2uNk++a3M0bddzeNM3a8Co92hAUczO7tSoe2eJXNZUMDedc20sKmjZnELUfitzaV0YVhWnyCBM68d4pLtBbrJud2SMAW1zQGaAenoM/OdNo4jswObLnqp13rF3JLy2HXJ/vFJ/t7z46re3Y8BY0cs1Ct/l+zCvv27lrylRvLeRyI+3XM5Vs7tOQPLchwP5nJZEMN9EBwZ6KuY1IntBsKSvS+RHdDuuZgx4L6Dv0d17MgkDw/dF1vkgpULQZDxWdj6VaS0paS9pFPTzfyuO4ifp6A0Y81NpbqP5X9II8jLnUk6bdha39VMbRy4nuI7kmMFWIXaFggkTC+UiMS3V1GOhefx8BEUwjDdiXaVGPxsjeMse91AfunJEWo/6HsvrgS5pZ8xO02hb8745UyLCJa3OMSAUBrfKYG7yhJ6OuDVKhw7q7TpavLXMkPdLsWgso3reEtkG5wc4LRKtfVwA32UjJadhv1D96q4bGZ+T8xfnmuECJiCINrlsS6bd79D9fyhhzAf/usuEl7f4B8q1hVxQRw5E/WvxSMXl31A2ugggwenkS6MKv5ZkQHC2VzfPdtIcAfabsDdthCbwlHPBMmU7coQ00TBDW90LL8eKpkiWp9YlQ1w==
-    ADMIN_USERNAME: AgAjz/v2LyfQbrY20FaIpP5jpr1Gu+4KdbvffuVsTiUTqSghOTKViteqAkw0J7OytyXfShIvrD3hiaKQnZG7eQnnwgCtOB1LYT8apARtzvCeoL4Ysr94mUYh/JRtRtSf1K0zI5hEzWBs8c8EcyDaw0ch+7TTZw68ih52OJVpN8AIpkpvi10M97cnp3IzguNEJV0ex6O8s5f2G10s7elUnIZzkdtvYL+1aEeu7HknCITIUhvQkF11iGarV5FAr2vY7gpBwzjcnbW3Sz0Bv9Pst1cNfKyHitUoeql3fL+Jf0rKMIQDnNrCRcWk1gMYzZrMpcD4PAP8o7CygLw8CTNVeod9dv0gfJVOmiQVxRbC7y7ZDm672e61M+kc3mnHtASAI1XRI2IBKCTg/jmjhFF4hUu3p2pjYS/azkJDu3wEqteAtDBG6sd44v8DsJrDDASLN4fGdhaQ3UZNK6hGTgjyVzI3qugfu+v3DPv3zVIXfRRqMMiDCTOc+8dX55eWm/SarBfV8cUZ80RlFV8XMGU49fxPzCmRus0ZZX/pyr/Gxq67ZVQupyswQh1I+PXFo00bF9/BahdffYGIORo3Qd9Ng3JnewaIu0VsJHWtWSEGVgO9s/iX3V7ZT84wjalFznK0IJNQCb+J9dvfub2zFh9ShvJ9lPq+mGlYk0SXC707EQK6aESPbCErBCd+yZFF2WMem8ofWIon9w==
+    ADMIN_USERNAME: AgCh0p8Mff/oP++z2Y0cCe7yKeL9+vyklylTLyRKxjluCAcpuIyzqwpCdmWmN1LvCd8hn68UbeKXRSGt8WC0W/AzQ2PzPNpQ0kQCUhFB4N2Mv/UOxT7meXSO+byYYyCR5AyJ3xL5DqVrHXlzbugxkNLcROI5YUwSnKgX4vWrBO4Emklp7YVYDIut3spueE9HiZ8aN/5iV+Cxsw4b3D229rHWqi8NWzWie7xeAU/IScm8B/tbyLYC/EstRKhcThMDiyd8pm5jM/5ubJgAm1P5n0Q1XGYrAtiOnRTDqzGGPGJ7skMDGlYii0z0U3Nb1US7/4Sen0uHXyHIgn8JptQTxAqACE8eFMw0A9Nm+8Ms1ZsxmhnCMQx3bljk0hB9IXBrTQ5rO35+fRVahsTS3aIW1WK1b+2EbHEk5djWheWEml8X6pC41dNH887dc0xB/1pRNlw5mD9z/SRjQRB+rAYPrCzW/GDI43xnGxrgWN6bEiPH+CorqVj8ahIRUffDcI7ZCve6EuXcyeTMGeo5rTg9s7my2KWQCzoXUAJFaNBTbzxhXtr3Rx+l1YtR4zRXcPJQxReu1hMcPhJhJY2rP3M8nUgUZoJY1SbK9tRNWJqurcV4nhyd/w5t67cv+M1uxOTtllsJv5Wg4OMiZAZrKzAgtxJIrM2G/lXTatk02YmFiAYbCEsm/r0IQ8pov4hsJI5ebZyoCWOOgA==
-    DATABASE_URL: AgAxGhBom5Z6+NxnIoTHaP05BDXJaVC2k1kUUe7/jmj0d15hsRR2so5NSqHpfZEYkOKQ484hu0Om+USrDyFr73BqcgHMGa1/WqiwuxMtw9/rh26zEONl0aRNrQponOngYigMBq0+zVBUGm8i5i6A03D3auyX0ww1kZO9yKqJmKnk2+GV5x6BnnDAfkVdn0v88kkdhuVUiXz5UH98PA+lJU7FtwR06uUYkKCPtc0JChM5jzajKqqqOhcgUW3d7I7ATqVeGWk3wn04uHFjRtxvYhlF6EF6wU4VVmEjwOQJ5aWLPEZuc2UpOhTjb6FtK/CNphVjUzCGSIF9WdZT6TacBVf2q88Zc1Gan2Mm3bE1OtBGek0qK94X1X+OgtwYYBPlZTTGMDBSF6qkb0QnYmTOuNKRC50IBxuGwFwh6T+V14ttEHUhipOknDbxdRzVpo1vvKqDORNcgZaenyK4fCpjU5xyYKMOR65YmubPLm7jL/0hwZvkBJjLfszjeqbKD+ptrZEmgvrALyHLxYEjXfVMblEKltME5vIj6DoM2IEHM1P8kTnDtfoN+YtOTPO9xtnHh1VSxV5QH9v82HeFVVZ0jT4oudAL0uL6bA2OnJMgFue7yrSYepJZhUdldndYQP0k6ur9aeuAwp6ptteif/fDFzsQ++qF8FwuZVJ7T/tw4IyyBL4dnieTaCISoHUEi1P3BMB16hDVsqnwXsmc2L1bo6aCLQ5ySVidfIbRdHz76VUJVsZqvfTIO+mXTTwvJexOEBxnNWMtwF+oQYIW4US8Two+JvM=
+    DATABASE_URL: AgCbCyCwXzkW/6gd4XWo+/xyentimc1ERwBXyutbdwW6zT5Z9HqljwGC9u2I3wScAmE1X9IiBvQU2n6+ESqAHtlXpnIJBWEs9muppSR3BhIEhaLtfHCpX7DqxWYPKg0gdWtKK7wbizjRWffHmg1cO/xavjefseAMpvgWS6zaQu5GTXPC0jXgSpiD7kNrge5Fk5wHcJqilj8oTKgHqbAZfhkTMDzqhbxJ49VsI+ItfdGG/dPRXuDvJ9YAeD1/2vHjUBSffcSdAFf/Zgn/XI/YLOyw/zVrPRtm3Plyti0/9yPvHbh9K30uvAlf2H4+mwZkpiDKxrId1/VU1suHu3N7pB8CrmlgXsowJr95juzoF8lTvGSrg7kE4mQaIhE3hAQ5B3OiWnoaN9uHa0Htb4GhjTrsdWVwMxHfsRFVdSwWtITzm3miMuWbQfQ1SGZEI3TU0lexc8UYzbQBHCY5GgU92GTmEQAl84jLMeo+wyVYBIDdTL0sHmFZfv/omiW0lzScZChaKEkJGxRBsju3XG5f7MqLrohWg5uBm+riApEu46a4DGZUV3nNJZPdwnZbProvPTjb0VvAz7jfAFiznxVouS2+CKEH7/FMxkBKDnwagxfd7KAr0gE/RxmuFSq+G9nNoaBTWoEc2V3ViNUCrSlRW8yf19Ow0vP+MfHLWkZ09QjDRLKdxBZpOrda/mxrxFSeqYmK+XA6/uZAigdX3O0Rr8RHYNPLjDCfce37gquD04269Fk2goLS4xEzhiODpADMWFRxPFzw5KDLICXZL3BTEiHNhdYiEOEm53uz7aIG
-    OAUTH2_CLIENT_ID: AgB7Ksnv37BXR7GVKSgUbRGGc2+A1mbVWHxo9M3g83DEcPW3eWpJOvEtRTuPtRVFX8zZZPN9D37wCJbRT2rkfwBvP60TXvXmvV0jySSs/Sp6e1AAT/MS21BxmU4YL7OkQdIdfriv9QdXvX0wNM1ePjiVOhnJ1B3Oial+dtAp4Kw0KO81uQi/ils6ctQiZ365J0hh65z7YWtTUhzAbLLxUA8Rd19XBmotRPBMFhSXG2J4OvmSY1SoupeJkGt4Jo3pYs3e6g/Wggo113NUifNJb8tzSmGPM3fB8rckfiG+5A57h3/waAiKrRY71kutH4a+ATi0+5gQxVhZo+SkeeQD4Ws67ejH+/hNWAVst5Ld1xV2W5YRkVBXz/8dyDtm9fSaMTq+eoYOaQbGOMSjlI8dF4YZcd4Lkweecx8nl3ZVSChms+pUIVPztBNOSeophb04NA2U+sZ/HZHRXXQGm4WmSZRs3XDXZPNKBqPIbPasxE2+3vKkgdbWG9MP1s1vGI8Rc4kWsekt1nxMOTP6xE+A6/xcDz2Usjf+K2RZKWklvqSWwF3n9X0r5Do8+XXbII8QEjjY2XG1WIlsyKFTKzrGhcrkRtdQAz3hvCQ3735R414QSWvbD7rmvhF9+WPuamxjQu0F02N2TGK9t8LPuwGlSWfALkFhuetlHkrLKakcaULf/6V7MQjY1Ucdtr5lwzYbpJoluxnCJ8pTYULhVe3lyF7Dc9OeW9DdFTbBceFUEBTtD6aXxqaIiMLk
+    OAUTH2_CLIENT_ID: AgDOU++RfwYBBQ5zgbO0jbw3TkAzpUGRQmnk4f4pvqdgK+uiHOrjAEmgFUU4PRQ8Q/A0lZcYMP1XY0VRNGnIuqUqLv+xDBQY8+hwRD6lPI/eyw3mLSbWVzHyDm7szv4C1kxmls2hK4Y+T8sIjmquiekDgt9tGVeOO+jo6uVyU8TE80zKoxSUrk++WrviMVXkErtb7VVzJpRDVgU0xns0Ou/xg2t9P5BqzsrubvIrfVhDA7enNSXhsn6ZtjsNweZePGI9CXwTDNlMgqXi67H1Mrz6MwoMXXSRZSv8oRXDn/w8dD7D3DpJ/ZpUQN9WJBShTsqqIk2p8HJJvYUunbPyVaHZ6yCsUI/Lt8Y3kbfJPMjqjIRX9xMu28DstEUqvcCKxdBDghosVyf1v216VO+LMWiwuvF/n6UcCk9arTtejoxdkPz9txyS1Yyv7rqYd6wNEkXgBKmcqflQUQbblglfFgSJc//tQQ7+24LK86RsnO/aElsWu7hOi6TosZao+i3kJ18MYYd+jewJ31gTtlQs4jp3CIgFi1N7RVrp8zOerfe4GO8PdMWJ+CERyyfFtAAD5TSmY0n7F+663RP+GXacHTbhDv7QCqX9LGltq4Z1IcMXbSwqRiW+C1ShQkaaNnWarZQuNmFuihgUE0vCYIm8WiEIJe320X7sdUlQtQ6d/vFFSupgcZqwJLqb5xOagHNRlYcDEg2A9y2TnAIgFzSg4+6UdgMzYxrTgUnuUOz6eLJdbxRuQRsbS3a8
-    OAUTH2_CLIENT_SECRET: AgAjqVgKsaEJtlfAStw7Yq2Ohde3fP/RXqTUYiAVXxHBVHL/dp9k+20RhgMlAOjZ0seFwJymube7AKAba9/HNmLxGGbtcLR+Z0akAZ+gXaVsfUxfJAdUNVYhg2nRoAtZe0zN+JqwzUuIqbYDw9iDR6KyuEQg07JyEfxYHkoCOm1oyOGSvjwNkEPxp84AJQnXTRd5FoHo5FrepHsOv74lZ5PXC2eJblCmZ8EBT2b3AuQAmwQQd+Tz35IDZFNACXgAyBB8vHL/VmOawr+3Sf3EiaHXLuQA5+xnOko3JxKJyem/aZxrRafCUgnxQfKJ1N3+uebHj2v5RcHFh8d6P74IXC+dahq0WyIIdfLYsBv1qfR/vNT8JHE9d9G6X5FQkzJpEAcKqh+3lGYivFfyVZJzWEIaEeP30geRYksD/z0g1DGhj8XCfqjhnS9eiRZxB9hroPYyb/LtLB0vdgqDEA96adCN7qn6u9LlcsjC/fetSKuMi56RtPrAMzkpm56wlPNbA/E9DDnlNnK5lEwuXnxH7duB+i8DcDs5ResNGC3Z6kCs55tFRZQA1rTmtSGIdFBAcQSbQW5IbOk9I6HeVEX423MsglBwaVIVRFCswK0qTNIXj5u68/7drldEkmb84nhyXTOY+mRc9thTZiJASIO6HpsV+hTboHu9I+7yhSMxIyvYtjzBiL24elPg6L3L/3zlZqaxp8Nl9gkuU3rdtQnQYd0eTgripAvI8MONkC6mLUBqUoTFvg6XWGmDdsmnwO9zZ/I5vuszKpYFBAP1ErgJpk1djipmvciJMUrtxeLRKAT2PniwTx0mvQd2rrszya/ABz5wmDqcS6qeY1W2+laWpDRgrdAKukBIX4UoU2Xy0fm4Gg==
+    OAUTH2_CLIENT_SECRET: AgDJ8ClfSTL0vIoD19KLN7sJsslEnZivGvnzNo2fuTmKGYz1Z5e0YZfLwK40reJM+d9ECCrzCeGqQQe/0mxzmzdaciH6p0bLd0/7pLLfnuvy/ObGCrPtIXRojKPu0aGGvJwMs2f5wm24qE5jW2IT08vnNF25NFSYyDxUcefg3zpsfu/Ff3MZtjn9oaDsCxPLTfzGz3NXbHIqVi1b3Er5B/wFLqib28AIwoXwTHepdmJ0+8YcbP1GKuwADNdHre1LDgEMFcgF0AuhlMUEVEKJtrKczdF59HOg2lwPQaHcZEVy1F1hgqBCbQSkI4pVfZ99pmpHRnn9PytNd+3tktdpu9So3TlR+GM8BON/2utlYdDagPBlctdJ+7mwG+PsptG/RSWw6cUqHvTIJp7wcMWT/vqyPsHeXoZy4x9TpLad6EQQHU969+fZN/Sz6d/vpjaqbbkqLACJ/e1PVBHBot/D3y/sCDovebSnJ8+SMsvhrc1kjOZhs5oFcipHxwObSyyJ8sah+Rbz+MgbWNBZ6uuPPEay5T/V12lrT4WfGoMnc1fWe9WDSMXt/GiFCvFhHBJfhcVjaKQ4OZT+TSo4E+guU7B1YvbY6+A7zwke2bdBoSluV10DTo87pp2tCY+x4AKsP1WVIubXa/ck91WsDM44WPYsaZ/F+FkD8P/gFk3eJmGQQx5qOmgzSNldwWtuj74H3TprXId+J22a/La4Qrz0JE/S6Arkfh88UiMCmF0ATb2zwuEYam4FKyKa3iiNMvo/WzEZwIlsCSE3tB2KvnC4Gf6rYO2Ef72fu7jNov8w1Ctu4rgcAk3IrUw3awoT7V3KANOnPMUu+Xp+tVNIXxJv1aAvY+HgR8XQ/h0uFMi556kkgQ==
   template:
     metadata:
       creationTimestamp: null

k8s/services/system-upgrade/plan.yaml

@@ -0,0 +1,50 @@
+# These plans are adapted from work by Dax McDonald (https://github.com/daxmc99) and Hussein Galal (https://github.com/galal-hussein)
+# in support of Rancher v2 managed k3s upgrades. See Also: https://rancher.com/docs/k3s/latest/en/upgrades/automated/
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-server
+  namespace: system-upgrade
+  labels:
+    k3s-upgrade: server
+spec:
+  concurrency: 1 # Batch size (roughly maps to maximum number of unschedulable nodes)
+  version: v1.20.11+k3s1
+  nodeSelector:
+    matchExpressions:
+      - {key: k3s-upgrade, operator: Exists}
+      - {key: k3s-upgrade, operator: NotIn, values: ["disabled", "false"]}
+      - {key: k3os.io/mode, operator: DoesNotExist}
+      - {key: node-role.kubernetes.io/control-plane, operator: Exists}
+  serviceAccountName: system-upgrade
+  cordon: true
+  upgrade:
+    image: rancher/k3s-upgrade
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-agent
+  namespace: system-upgrade
+  labels:
+    k3s-upgrade: agent
+spec:
+  concurrency: 2 # Batch size (roughly maps to maximum number of unschedulable nodes)
+  version: v1.20.11+k3s1
+  nodeSelector:
+    matchExpressions:
+      - {key: k3s-upgrade, operator: Exists}
+      - {key: k3s-upgrade, operator: NotIn, values: ["disabled", "false"]}
+      - {key: k3os.io/mode, operator: DoesNotExist}
+      - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist}
+  serviceAccountName: system-upgrade
+  prepare:
+    # Defaults to the same "resolved" tag that is used for the `upgrade` container, NOT `latest`
+    image: rancher/k3s-upgrade
+    args: ["prepare", "k3s-server"]
+  drain:
+    force: true
+    skipWaitForDeleteTimeout: 60 # 1.18+ (honor pod disruption budgets up to 60 seconds per pod then moves on)
+  upgrade:
+    image: rancher/k3s-upgrade

tofu/proxmox/main.tf

@@ -22,7 +22,8 @@ module "master1" {
   vm_name    = "master1.ramiel.fuku"
   node_name  = "ramiel"
   ipconfig0  = "ip=192.168.1.31/24,gw=192.168.1.1"
-  memory     = 4096
+  memory     = 5120
+  disk_storage_name = "storage"
   core_count = 2
 }