feat(core): add region-level domain failover with circuit breaker

core/CMakeLists.txt

@@ -20,9 +20,11 @@ set(core_public_header
     include/tencentcloud/core/AbstractClient.h
     include/tencentcloud/core/AbstractModel.h
     include/tencentcloud/core/AsyncCallerContext.h
+    include/tencentcloud/core/CircuitBreaker.h
     include/tencentcloud/core/CommonClient.h
     include/tencentcloud/core/Config.h
     include/tencentcloud/core/Credential.h
+    include/tencentcloud/core/DomainFailoverManager.h
     include/tencentcloud/core/Error.h
     include/tencentcloud/core/Executor.h
     include/tencentcloud/core/NetworkProxy.h
@@ -43,6 +45,7 @@ set(core_public_header_http
 set(core_public_header_profile
     include/tencentcloud/core/profile/ClientProfile.h
     include/tencentcloud/core/profile/HttpProfile.h
+    include/tencentcloud/core/profile/RegionBreakerProfile.h
     )
 
 set(core_public_header_utils
@@ -57,8 +60,10 @@ set(core_src
     src/AbstractClient.cpp
     src/AbstractModel.cpp
     src/AsyncCallerContext.cpp
+    src/CircuitBreaker.cpp
     src/CommonClient.cpp
     src/Credential.cpp
+    src/DomainFailoverManager.cpp
     src/Executor.cpp
     src/NetworkProxy.cpp
     src/Sign.cpp

core/include/tencentcloud/core/AbstractClient.h

@@ -21,12 +21,35 @@
 #include <tencentcloud/core/AsyncCallerContext.h>
 #include <tencentcloud/core/profile/ClientProfile.h>
 #include <tencentcloud/core/http/HttpClient.h>
+#include <tencentcloud/core/DomainFailoverManager.h>
+#include <tencentcloud/core/CircuitBreaker.h>
 #include "AbstractModel.h"
 #include <map>
+#include <memory>
+#include <mutex>
 #include <unordered_map>
+#include <vector>
 
 namespace TencentCloud
 {
+    /// Abstract base for all generated service clients.
+    ///
+    /// Thread safety & lifetime contract:
+    ///
+    ///   1. Set*() methods (SetClientProfile, SetCredential,
+    ///      SetNetworkProxy, SetHeader, SetRegion) are NOT thread-safe
+    ///      with concurrent DoRequest / DoRequestAsync calls. Configure
+    ///      the client fully before issuing requests from multiple
+    ///      threads.
+    ///
+    ///   2. DoRequestAsync() dispatches work on HttpClient's background
+    ///      worker; its completion callback captures the selected
+    ///      breaker as a std::shared_ptr<CircuitBreaker> (NOT `this`).
+    ///      The breaker therefore stays alive for the whole callback
+    ///      regardless of AbstractClient's destruction timing, so the
+    ///      callback never dereferences a dangling client. The
+    ///      destructor still deletes m_httpClient first to join the
+    ///      worker before other members are torn down.
     class AbstractClient
     {
     public:
@@ -62,6 +85,28 @@ namespace TencentCloud
         template <typename Req, typename Resp>
         void DoRequestAsync(std::string action, Req req, ReqOpts opts, AsyncCompletionHandler<Req, Resp> handler);
 
+        // Result of picking an endpoint for one request.
+        struct EndpointDecision
+        {
+            std::string host;  // endpoint to send to (always non-empty)
+            // Breaker to report the outcome to; nullptr when bypassed
+            // (failover disabled / non-TencentCloud) or when falling
+            // through to the bottom fallback (force-send, no report).
+            std::shared_ptr<CircuitBreaker> breaker;
+        };
+
+        // Get-or-create the breaker guarding |host| under |origin|.
+        std::shared_ptr<CircuitBreaker> BreakerFor(const std::string &origin,
+                                                   const std::string &host);
+
+        // Pick the endpoint for this request. See EndpointDecision.
+        EndpointDecision SelectEndpoint(const std::string &primary_endpoint);
+
+        // Report the outcome to the breaker that Allow()ed this request.
+        // |breaker| == nullptr is a no-op (bypass / bottom fallback).
+        static void ReportResult(const std::shared_ptr<CircuitBreaker> &breaker,
+                                 bool success);
+
     private:
         Credential m_credential;
         ClientProfile m_clientProfile;
@@ -72,6 +117,37 @@ namespace TencentCloud
         HttpClient *m_httpClient;
         std::string m_service;
         std::map<std::string, std::string> m_headers;
+
+        // Region failover (domain-level circuit breaker).
+        //
+        // Breakers are keyed by (origin, host) in a lazily-populated
+        // registry. The candidate list for a request is built by
+        // DomainFailoverManager::BuildCandidates(); every candidate
+        // except the LAST is guarded by a breaker. The last candidate is
+        // the bottom fallback: it is force-sent without a breaker and
+        // without reporting (legacy behavior preserved).
+        //
+        // shared_ptr is used so async completion callbacks can capture
+        // the selected breaker directly; the breaker then outlives the
+        // AbstractClient if needed, independent of destruction order.
+        struct BreakerRegistry
+        {
+            std::mutex mutex;  // guards lazy insertion into |map|
+            // key = origin + "\n" + host
+            std::map<std::string, std::shared_ptr<CircuitBreaker> > map;
+        };
+
+        // nullptr when region failover is disabled -> fully bypassed.
+        std::shared_ptr<BreakerRegistry> m_breakers;
+
+        void InitRegionBreakers();
+
+        static bool IsFailoverTriggering(const std::string &error_code);
+
+        HttpClient::HttpResponseOutcome DoRequestWithEndpoint(
+            const std::string &actionName, const std::string &body,
+            std::map<std::string, std::string> &headers,
+            const std::string &endpoint);
     };
 }
 
@@ -81,25 +157,34 @@ void TencentCloud::AbstractClient::DoRequestAsync(
 {
     using RequestOutcome = Outcome<Core::Error, Resp>;
     const auto& http_profile = m_clientProfile.GetHttpProfile();
-    std::string endpoint = http_profile.GetEndpoint();
-    if (endpoint.empty())
+    std::string primary_endpoint = http_profile.GetEndpoint();
+    if (primary_endpoint.empty())
     {
-        endpoint = m_endpoint;
+        primary_endpoint = m_endpoint;
     }
 
-    std::string::size_type pos = endpoint.find_first_of('.');
+    // Pick an endpoint for this request based on circuit breaker state.
+    EndpointDecision decision = SelectEndpoint(primary_endpoint);
+    std::string resolved_endpoint = decision.host;
+    std::shared_ptr<CircuitBreaker> breaker = decision.breaker;
+
+    std::string::size_type pos = resolved_endpoint.find_first_of('.');
     if (pos != std::string::npos)
-        m_service = endpoint.substr(0, pos);
+        m_service = resolved_endpoint.substr(0, pos);
     else
     {
         m_service = "unknown";
-        Core::Error err("ClientError", "endpoint `" + endpoint + "` is not valid");
+        // Endpoint is syntactically invalid and cannot be used.
+        // Release the HalfOpen probe slot SelectEndpoint reserved, so
+        // invalid-endpoint paths don't leak breaker probe capacity.
+        ReportResult(breaker, /*success=*/false);
+        Core::Error err("ClientError", "endpoint `" + resolved_endpoint + "` is not valid");
         handler(req, RequestOutcome(err));
         return;
     }
 
     Url url;
-    url.SetHost(endpoint);
+    url.SetHost(resolved_endpoint);
     HttpProfile::Scheme scheme = http_profile.GetProtocol();
     if (scheme == HttpProfile::Scheme::HTTP)
         url.SetScheme("http");
@@ -145,8 +230,29 @@ void TencentCloud::AbstractClient::DoRequestAsync(
     m_httpClient->SetReqTimeout(http_profile.GetReqTimeout() * 1000);
     m_httpClient->SetConnectTimeout(http_profile.GetConnectTimeout() * 1000);
 
-    m_httpClient->SendRequestAsync(http_req, [req, handler](HttpClient::HttpResponseOutcome http_resp)
+    // Align with the synchronous path: honor user-provided CA and
+    // resolve-IP settings so that async requests respect HTTPS custom
+    // trust anchors and forced DNS resolution.
+    m_httpClient->SetCaInfo(http_profile.GetCaInfo());
+    m_httpClient->SetCaPath(http_profile.GetCaPath());
+    m_httpClient->SetResolveIp(http_profile.GetResolveIp());
+
+    // Capture the selected breaker (a shared_ptr) so the async callback
+    // can Report() to it. The breaker is kept alive by the captured
+    // shared_ptr for the whole callback, independent of AbstractClient's
+    // destruction order. The callback no longer captures `this`.
+    m_httpClient->SendRequestAsync(http_req,
+        [req, handler, breaker](
+            HttpClient::HttpResponseOutcome http_resp) mutable
     {
+        // Unified report rule: any Allow()==true request MUST Report()
+        // exactly once. A non-failover error (business 4xx / throttling)
+        // is treated as a "success" for the endpoint, so the HalfOpen
+        // probe slot is released (fixes P7). |breaker| nullptr -> no-op.
+        bool ok = http_resp.IsSuccess() ||
+                  !IsFailoverTriggering(http_resp.GetError().GetErrorCode());
+        ReportResult(breaker, ok);
+
         if (!http_resp.IsSuccess())
         {
             handler(req, RequestOutcome(http_resp.GetError()));

core/include/tencentcloud/core/CircuitBreaker.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2017-2019 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TENCENTCLOUD_CORE_CIRCUITBREAKER_H_
+#define TENCENTCLOUD_CORE_CIRCUITBREAKER_H_
+
+#include <cstdint>
+#include <mutex>
+
+#include <tencentcloud/core/profile/RegionBreakerProfile.h>
+
+namespace TencentCloud
+{
+
+/// Three-state circuit breaker for region failover.
+///
+///   Closed   -> traffic allowed; failures counted in a sliding window
+///   Open     -> traffic rejected; after |timeout| enter HalfOpen
+///   HalfOpen -> at most |max_requests| concurrent probes; enough
+///               successes close it, any failure reopens it
+///
+/// Usage:
+///   if (breaker.Allow()) {
+///     bool ok = SendRequest();
+///     breaker.Report(ok);   // MUST pair with every Allow() == true
+///   }
+class CircuitBreaker
+{
+public:
+    enum class State
+    {
+        kClosed,
+        kOpen,
+        kHalfOpen,
+    };
+
+    CircuitBreaker();
+    explicit CircuitBreaker(const RegionBreakerProfile &profile);
+
+    // Holds a std::mutex -- non-copyable, non-movable.
+    CircuitBreaker(const CircuitBreaker &) = delete;
+    CircuitBreaker &operator=(const CircuitBreaker &) = delete;
+
+    /// Returns true iff a new request should be allowed to proceed.
+    /// On true, the caller MUST subsequently call Report() with the
+    /// outcome; otherwise HalfOpen probe slots leak.
+    bool Allow();
+
+    /// Report the outcome of a request that got Allow() == true.
+    void Report(bool success);
+
+    /// Observational.
+    State GetState();
+
+private:
+    void LazyAdvanceStateLocked();
+    void TransitionToLocked(State new_state);
+    bool ReadyToOpenLocked() const;
+    int64_t NowMs() const;
+
+    std::mutex m_mutex;
+    State m_state;
+
+    // Counters within the current state window; reset on any state
+    // transition and on Closed-window expiry.
+    int m_total;
+    int m_failures;
+    int m_consecutiveFailures;
+
+    // HalfOpen-only: number of probes Allow()ed but not yet Report()ed.
+    // Bounded by |m_maxRequests| to avoid flooding a fragile endpoint.
+    int m_halfOpenInFlight;
+
+    // Absolute timestamp (ms) at which the current window/timeout
+    // expires. 0 means "no auto timeout" (HalfOpen).
+    int64_t m_expiryMs;
+
+    // Immutable configuration (derived from RegionBreakerProfile).
+    int m_maxFailNum;
+    double m_maxFailPercent;
+    int64_t m_windowIntervalMs;
+    int64_t m_timeoutMs;
+    int m_maxRequests;
+};
+
+}  // namespace TencentCloud
+
+#endif  // TENCENTCLOUD_CORE_CIRCUITBREAKER_H_