(WIP) (elqui) rook bucket support & nfs

fleet/lib/fleet-conf/overlays/cp/gitrepo-elqui.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: fleet-default
 spec:
   repo: https://github.com/lsst-it/k8s-cookbook
-  branch: cp_production
+  branch: IT-5665/ceph-tuning
   keepResources: true
   paths:
     - fleet/s/cp/c/elqui/*

fleet/lib/rook-ceph-cluster/overlays/elqui/values.yaml

@@ -7,6 +7,7 @@ cephClusterSpec:
       osd_pool_default_pg_autoscale_mode: warn
       rgw_override_bucket_index_max_shards: "401"
       rgw_enable_usage_log: "false"
+      rgw_multipart_min_part_size: "1048576"
     mgr:
       mgr/balancer/upmap_max_deviation: "1"
     osd:

fleet/lib/rook-ceph-conf/charts/elqui/templates/cephnfs-obs-env.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+  name: obs-env
+  namespace: rook-ceph
+spec:
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: 3
+    quotas:
+      maxSize: 10Gi
+    parameters:
+      nodelete: "true"
+      nosizechange: "true"
+      pg_autoscale_mode: "off"
+      pg_num: "64"
+  dataPools:
+    - failureDomain: host
+      replicated:
+        size: 3
+      quotas:
+        maxSize: 250Gi
+      parameters:
+        nodelete: "true"
+        nosizechange: "true"
+        pg_autoscale_mode: "off"
+        pg_num: "128"
+  metadataServer:
+    activeCount: 1
+    activeStandby: true
+    resources:
+      limits:
+        cpu: "4"
+        memory: 4Gi
+      requests:
+        cpu: "1"
+        memory: 4Gi
+  preserveFilesystemOnDelete: false
+---
+apiVersion: ceph.rook.io/v1
+kind: CephNFS
+metadata:
+  name: obs-env
+  namespace: rook-ceph
+spec:
+  rados:
+    pool: obs-env-data0
+  server:
+    active: 1
+    resources:
+      limits:
+        cpu: "3"
+        memory: 8Gi
+      requests:
+        cpu: "1"
+        memory: 8Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rook-ceph-nfs
+    ceph_daemon_type: nfs
+    ceph_nfs: obs-env
+    instance: a
+    rook_cluster: rook-ceph
+  name: rook-ceph-nfs-obs-env
+  namespace: rook-ceph
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 139.229.181.22
+spec:
+  ports:
+    - name: nfs
+      port: 2049
+      protocol: TCP
+      targetPort: 2049
+  selector:
+    app: rook-ceph-nfs
+    ceph_daemon_type: nfs
+    ceph_nfs: obs-env
+    instance: a
+    rook_cluster: rook-ceph
+  type: LoadBalancer

fleet/lib/rook-ceph-conf/charts/elqui/templates/cm-cephcli.yaml

@@ -10,8 +10,6 @@ data:
     ceph orch set backend ""
     ceph mgr module disable rook
 
-    ceph mgr module enable rook
-    ceph orch set backend rook
     ceph device monitoring on
     ceph config set global device_failure_prediction_mode local
     ceph telemetry on --license sharing-1-0

fleet/lib/rook-ceph-conf/charts/elqui/templates/objectbucketclaim-rubinobs-butler-latiss.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: rubinobs-butler-latiss
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: rubinobs-butler-latiss
+  namespace: rook-ceph
+spec:
+  bucketName: rubinobs-butler-latiss
+  storageClassName: rubinobs-butler-latiss
+  additionalConfig:
+    maxSize: 1Ti  # quota for BTS; XXX increase for cp

fleet/lib/rook-ceph-conf/charts/elqui/templates/objectbucketclaim-rubinobs-butler-lsstcam.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: rubinobs-butler-lsstcam
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: rubinobs-butler-lsstcam
+  namespace: rook-ceph
+spec:
+  bucketName: rubinobs-butler-lsstcam
+  storageClassName: rubinobs-butler-lsstcam
+  additionalConfig:
+    maxSize: 34Ti  # quota for BTS; XXX increase for cp

fleet/lib/rook-ceph-conf/charts/elqui/templates/objectbucketclaim-rubinobs-raw-latiss.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: rubinobs-raw-latiss
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: rubinobs-raw-latiss
+  namespace: rook-ceph
+spec:
+  bucketName: rubinobs-raw-latiss
+  storageClassName: rubinobs-raw-latiss
+  additionalConfig:
+    maxSize: 1Ti  # quota for BTS; XXX increase for cp

fleet/lib/rook-ceph-conf/charts/elqui/templates/objectbucketclaim-rubinobs-raw-lsstcam.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: rubinobs-raw-lsstcam
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: rubinobs-raw-lsstcam
+  namespace: rook-ceph
+spec:
+  bucketName: rubinobs-raw-lsstcam
+  storageClassName: rubinobs-raw-lsstcam
+  additionalConfig:
+    maxSize: 6Ti  # quota for BTS; XXX increase for cp

fleet/lib/rook-ceph/values.yaml

@@ -26,6 +26,8 @@ csi:
   cephcsi:
     image: quay.io/cephcsi/cephcsi:v3.12.2
   enableLiveness: true
+  nfs:
+    enabled: true
   provisionerTolerations:
     - <<: *storage_node_tol
   provisionerNodeAffinity: *storage_node_aff