CASSANDRA-19195 Added guardrail to enforce minimum CMS size

CHANGES.txt

@@ -1,4 +1,5 @@
 7.0
+ * Add a guardrail to disallow reconfiguring CMS below a minimum size (CASSANDRA-19195)
  * Don't increment client metrics on messaging service connection unpause (CASSANDRA-21491)
  * Add nodetool getreplicas (CASSANDRA-17665)
  * Implementation of CEP-49: Hardware-accelerated compression (CASSANDRA-20975)

conf/cassandra.yaml

@@ -2585,6 +2585,12 @@ drop_compact_storage_enabled: false
 # maximum_replication_factor_warn_threshold: -1
 # maximum_replication_factor_fail_threshold: -1
 #
+
+# Guardrail to fail an operator-requested CMS reconfiguration (nodetool cms reconfigure / JMX) when the resulting
+# CMS size would be below this threshold. The CMS size is the aggregate replication factor across all datacenters.
+# The check is skipped when the cluster has fewer nodes than the threshold. Defaults to -1 to disable.
+# minimum_cms_size_fail_threshold: -1
+
 # Guardrail to warn or fail when a statement is prepared without bind markers (parameters).
 # This prevents the Prepared Statement Cache from being flooded with unique entries caused
 # by hardcoded literals. When warned is true, logs a warning on the usage of misprepared statements.

conf/cassandra_latest.yaml

@@ -2359,6 +2359,11 @@ drop_compact_storage_enabled: false
 # maximum_replication_factor_warn_threshold: -1
 # maximum_replication_factor_fail_threshold: -1
 #
+# Guardrail to fail an operator-requested CMS reconfiguration (nodetool cms reconfigure / JMX) when the resulting
+# CMS size would be below this threshold. The CMS size is the aggregate replication factor across all datacenters.
+# The check is skipped when the cluster has fewer nodes than the threshold. Defaults to -1 to disable.
+# minimum_cms_size_fail_threshold: -1
+#
 # Guardrail to warn or fail when a statement is prepared without bind markers (parameters).
 # This prevents the Prepared Statement Cache from being flooded with unique entries caused
 # by hardcoded literals. When warned is true, logs a warning on the usage of misprepared statements.

src/java/org/apache/cassandra/config/Config.java

@@ -1056,6 +1056,7 @@ public static void setClientMode(boolean clientMode)
     public volatile int minimum_replication_factor_fail_threshold = -1;
     public volatile int maximum_replication_factor_warn_threshold = -1;
     public volatile int maximum_replication_factor_fail_threshold = -1;
+    public volatile int minimum_cms_size_fail_threshold = -1;
     public volatile boolean zero_ttl_on_twcs_warned = true;
     public volatile boolean zero_ttl_on_twcs_enabled = true;
     public volatile boolean non_partition_restricted_index_query_enabled = true;

src/java/org/apache/cassandra/config/GuardrailsOptions.java

@@ -110,6 +110,7 @@ public GuardrailsOptions(Config config)
         validateDataDiskUsageMaxDiskSize(config.data_disk_usage_max_disk_size);
         validateMinRFThreshold(config.minimum_replication_factor_warn_threshold, config.minimum_replication_factor_fail_threshold);
         validateMaxRFThreshold(config.maximum_replication_factor_warn_threshold, config.maximum_replication_factor_fail_threshold);
+        validateMinCmsSizeThreshold(config.minimum_cms_size_fail_threshold, "minimum_cms_size_fail_threshold");
         validateTimestampThreshold(config.maximum_timestamp_warn_threshold, config.maximum_timestamp_fail_threshold, "maximum_timestamp");
         validateTimestampThreshold(config.minimum_timestamp_warn_threshold, config.minimum_timestamp_fail_threshold, "minimum_timestamp");
         validateMaxLongThreshold(config.sai_sstable_indexes_per_query_warn_threshold,
@@ -1051,6 +1052,21 @@ public void setMinimumReplicationFactorThreshold(int warn, int fail)
                                   x -> config.minimum_replication_factor_fail_threshold = x);
     }
 
+    @Override
+    public int getMinimumCmsSizeFailThreshold()
+    {
+        return config.minimum_cms_size_fail_threshold;
+    }
+
+    public void setMinimumCmsSizeFailThreshold(int fail)
+    {
+        validateMinCmsSizeThreshold(fail, "minimum_cms_size_fail_threshold");
+        updatePropertyWithLogging("minimum_cms_size_fail_threshold",
+                                  fail,
+                                  () -> config.minimum_cms_size_fail_threshold,
+                                  x -> config.minimum_cms_size_fail_threshold = x);
+    }
+
     @Override
     public int getMaximumReplicationFactorWarnThreshold()
     {
@@ -1521,6 +1537,17 @@ private static void validateMaxRFThreshold(int warn, int fail)
                                                       fail, DatabaseDescriptor.getDefaultKeyspaceRF()));
     }
 
+    private static void validateMinCmsSizeThreshold(int value, String name)
+    {
+        if (value == -1)
+            return;
+
+        if (value < 3)
+            throw new IllegalArgumentException(format("Invalid value %d for %s: minimum allowed value is 3, " +
+                                                      "as CMS requires at least 3 replicas to maintain quorum safely. " +
+                                                      "Use -1 to disable this guardrail", value, name));
+    }
+
     public static void validateTimestampThreshold(DurationSpec.LongMicrosecondsBound warn,
                                                   DurationSpec.LongMicrosecondsBound fail,
                                                   String name)

src/java/org/apache/cassandra/db/guardrails/CMSSizeGuardrail.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.function.ToLongFunction;
+
+import org.apache.cassandra.service.ClientState;
+
+import static java.lang.String.format;
+
+public class CMSSizeGuardrail extends MinThreshold
+{
+    /**
+     * Creates a new minimum threshold guardrail.
+     *
+     * @param failThreshold   a {@link ClientState}-based provider of the value above which the operation should be aborted.
+     */
+    public CMSSizeGuardrail(ToLongFunction<ClientState> failThreshold)
+    {
+        super("minimum_cms_size",
+              null,
+              state -> -1,
+              failThreshold,
+              (isWarning, what, value, threshold) ->
+              format("The CMS size of %s is below the failure threshold of %s. " +
+                     "Reconfigure CMS so its total size (sum of replication factors across all datacenters) is at least %s.",
+                     value, threshold, threshold));
+        this.throwOnNullClientState = true;
+    }
+
+    public void guard(int totalNodes, int cmsSize)
+    {
+        // If the request already uses every node, the cluster can't do better, so skip the check.
+        // Otherwise the CMS size must meet the configured minimum.
+        if (cmsSize >= totalNodes)
+            return;
+
+        guard(cmsSize, "CMS", false, null);
+    }
+}

src/java/org/apache/cassandra/db/guardrails/Guardrails.java

@@ -615,6 +615,12 @@ public final class Guardrails implements GuardrailsMBean
                      format("The keyspace %s has a replication factor of %s, below the %s threshold of %s.",
                             what, value, isWarning ? "warning" : "failure", threshold));
 
+    /**
+     * Guardrail on the minimum CMS size (aggregate replication factor across DCs).
+     */
+    public static final CMSSizeGuardrail minimumCmsSize =
+    new CMSSizeGuardrail(state -> CONFIG_PROVIDER.getOrCreate(state).getMinimumCmsSizeFailThreshold());
+
     /**
      * Guardrail on the maximum replication factor.
      */
@@ -1670,6 +1676,18 @@ public void setMinimumReplicationFactorThreshold(int warn, int fail)
         DEFAULT_CONFIG.setMinimumReplicationFactorThreshold(warn, fail);
     }
 
+    @Override
+    public int getMinimumCmsSizeFailThreshold()
+    {
+        return DEFAULT_CONFIG.getMinimumCmsSizeFailThreshold();
+    }
+
+    @Override
+    public void setMinimumCmsSizeFailThreshold(int fail)
+    {
+        DEFAULT_CONFIG.setMinimumCmsSizeFailThreshold(fail);
+    }
+
     @Override
     public boolean getZeroTTLOnTWCSEnabled()
     {

src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java

@@ -466,6 +466,11 @@ public interface GuardrailsConfig
      */
     int getMinimumReplicationFactorFailThreshold();
 
+    /**
+     * @return The threshold to fail when the CMS size (aggregate replication factor across DCs) is below the threshold.
+     */
+    int getMinimumCmsSizeFailThreshold();
+
     /**
      * @return The threshold to warn when replication factor is greater than threshold.
      */

src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java

@@ -910,6 +910,17 @@ public interface GuardrailsMBean
      */
     void setMinimumReplicationFactorThreshold (int warn, int fail);
 
+    /**
+     * @return The threshold to fail when the CMS size (aggregate replication factor across DCs) is below the threshold.
+     */
+    int getMinimumCmsSizeFailThreshold();
+
+    /**
+     * @param fail The threshold to fail when the CMS size (aggregate replication factor across DCs) is below it.
+     *             -1 means disabled.
+     */
+    void setMinimumCmsSizeFailThreshold(int fail);
+
     /**
      * @return The threshold to fail when replication factor is greater than threshold.
      */

src/java/org/apache/cassandra/tcm/CMSOperations.java

@@ -33,6 +33,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.virtual.ClusterMetadataDirectoryTable;
 import org.apache.cassandra.db.virtual.ClusterMetadataLogTable;
 import org.apache.cassandra.schema.ReplicationParams;
@@ -162,13 +163,32 @@ public void resumeReconfigureCms()
     @Override
     public void reconfigureCMS(int rf)
     {
-        cms.reconfigureCMS(ReplicationParams.simpleMeta(rf, ClusterMetadata.current().directory.knownDatacenters()));
+        ReplicationParams params = ReplicationParams.simpleMeta(rf, ClusterMetadata.current().directory.knownDatacenters());
+        guardMinimumCmsSize(params);
+        cms.reconfigureCMS(params);
     }
 
     @Override
     public void reconfigureCMS(Map<String, Integer> rf)
     {
-        cms.reconfigureCMS(ReplicationParams.ntsMeta(rf));
+        ReplicationParams params = ReplicationParams.ntsMeta(rf);
+        guardMinimumCmsSize(params);
+        cms.reconfigureCMS(params);
+    }
+
+    /**
+     * Enforces the minimum CMS size guardrail against an operator-requested reconfiguration. This is only applied to
+     * explicit reconfigure requests (nodetool / JMX), not to the automatic reconfiguration that happens on topology
+     * changes, which reuses the already-committed replication params.
+     */
+    private void guardMinimumCmsSize(ReplicationParams params)
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        int totalNodes = metadata.directory.allJoinedEndpoints().size();
+        int cmsSize = params.options.values().stream()
+                                    .mapToInt(Integer::parseInt)
+                                    .sum();
+        Guardrails.minimumCmsSize.guard(totalNodes, cmsSize);
     }
 
     @Override