Torne (Richard Coles) | 5821806 | 2012-11-14 11:43:16 +0000 | [diff] [blame] | 1 | // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
| 5 | #include "net/url_request/url_request_throttler_manager.h" |
| 6 | |
| 7 | #include "base/logging.h" |
| 8 | #include "base/metrics/field_trial.h" |
| 9 | #include "base/metrics/histogram.h" |
Torne (Richard Coles) | 868fa2f | 2013-06-11 10:57:03 +0100 | [diff] [blame] | 10 | #include "base/strings/string_util.h" |
Torne (Richard Coles) | 5821806 | 2012-11-14 11:43:16 +0000 | [diff] [blame] | 11 | #include "net/base/net_log.h" |
| 12 | #include "net/base/net_util.h" |
| 13 | |
| 14 | namespace net { |
| 15 | |
| 16 | const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500; |
| 17 | const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200; |
| 18 | |
| 19 | URLRequestThrottlerManager::URLRequestThrottlerManager() |
| 20 | : requests_since_last_gc_(0), |
| 21 | enable_thread_checks_(false), |
| 22 | logged_for_localhost_disabled_(false), |
| 23 | registered_from_thread_(base::kInvalidThreadId) { |
| 24 | url_id_replacements_.ClearPassword(); |
| 25 | url_id_replacements_.ClearUsername(); |
| 26 | url_id_replacements_.ClearQuery(); |
| 27 | url_id_replacements_.ClearRef(); |
| 28 | |
| 29 | NetworkChangeNotifier::AddIPAddressObserver(this); |
| 30 | NetworkChangeNotifier::AddConnectionTypeObserver(this); |
| 31 | } |
| 32 | |
| 33 | URLRequestThrottlerManager::~URLRequestThrottlerManager() { |
| 34 | NetworkChangeNotifier::RemoveIPAddressObserver(this); |
| 35 | NetworkChangeNotifier::RemoveConnectionTypeObserver(this); |
| 36 | |
| 37 | // Since the manager object might conceivably go away before the |
| 38 | // entries, detach the entries' back-pointer to the manager. |
| 39 | UrlEntryMap::iterator i = url_entries_.begin(); |
| 40 | while (i != url_entries_.end()) { |
Torne (Richard Coles) | 868fa2f | 2013-06-11 10:57:03 +0100 | [diff] [blame] | 41 | if (i->second.get() != NULL) { |
Torne (Richard Coles) | 5821806 | 2012-11-14 11:43:16 +0000 | [diff] [blame] | 42 | i->second->DetachManager(); |
| 43 | } |
| 44 | ++i; |
| 45 | } |
| 46 | |
| 47 | // Delete all entries. |
| 48 | url_entries_.clear(); |
| 49 | } |
| 50 | |
| 51 | scoped_refptr<URLRequestThrottlerEntryInterface> |
| 52 | URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) { |
| 53 | DCHECK(!enable_thread_checks_ || CalledOnValidThread()); |
| 54 | |
| 55 | // Normalize the url. |
| 56 | std::string url_id = GetIdFromUrl(url); |
| 57 | |
| 58 | // Periodically garbage collect old entries. |
| 59 | GarbageCollectEntriesIfNecessary(); |
| 60 | |
| 61 | // Find the entry in the map or create a new NULL entry. |
| 62 | scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id]; |
| 63 | |
| 64 | // If the entry exists but could be garbage collected at this point, we |
| 65 | // start with a fresh entry so that we possibly back off a bit less |
| 66 | // aggressively (i.e. this resets the error count when the entry's URL |
| 67 | // hasn't been requested in long enough). |
| 68 | if (entry.get() && entry->IsEntryOutdated()) { |
| 69 | entry = NULL; |
| 70 | } |
| 71 | |
| 72 | // Create the entry if needed. |
| 73 | if (entry.get() == NULL) { |
| 74 | entry = new URLRequestThrottlerEntry(this, url_id); |
| 75 | |
| 76 | // We only disable back-off throttling on an entry that we have |
| 77 | // just constructed. This is to allow unit tests to explicitly override |
| 78 | // the entry for localhost URLs. Given that we do not attempt to |
| 79 | // disable throttling for entries already handed out (see comment |
| 80 | // in AddToOptOutList), this is not a problem. |
| 81 | std::string host = url.host(); |
| 82 | if (opt_out_hosts_.find(host) != opt_out_hosts_.end() || |
| 83 | IsLocalhost(host)) { |
| 84 | if (!logged_for_localhost_disabled_ && IsLocalhost(host)) { |
| 85 | logged_for_localhost_disabled_ = true; |
| 86 | net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST, |
| 87 | NetLog::StringCallback("host", &host)); |
| 88 | } |
| 89 | |
| 90 | // TODO(joi): Once sliding window is separate from back-off throttling, |
| 91 | // we can simply return a dummy implementation of |
| 92 | // URLRequestThrottlerEntryInterface here that never blocks anything (and |
| 93 | // not keep entries in url_entries_ for opted-out sites). |
| 94 | entry->DisableBackoffThrottling(); |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | return entry; |
| 99 | } |
| 100 | |
| 101 | void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) { |
| 102 | // There is an edge case here that we are not handling, to keep things |
| 103 | // simple. If a host starts adding the opt-out header to its responses |
| 104 | // after there are already one or more entries in url_entries_ for that |
| 105 | // host, the pre-existing entries may still perform back-off throttling. |
| 106 | // In practice, this would almost never occur. |
| 107 | if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) { |
| 108 | UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1); |
| 109 | |
| 110 | net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST, |
| 111 | NetLog::StringCallback("host", &host)); |
| 112 | opt_out_hosts_.insert(host); |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | void URLRequestThrottlerManager::OverrideEntryForTests( |
| 117 | const GURL& url, |
| 118 | URLRequestThrottlerEntry* entry) { |
| 119 | // Normalize the url. |
| 120 | std::string url_id = GetIdFromUrl(url); |
| 121 | |
| 122 | // Periodically garbage collect old entries. |
| 123 | GarbageCollectEntriesIfNecessary(); |
| 124 | |
| 125 | url_entries_[url_id] = entry; |
| 126 | } |
| 127 | |
| 128 | void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) { |
| 129 | // Normalize the url. |
| 130 | std::string url_id = GetIdFromUrl(url); |
| 131 | url_entries_.erase(url_id); |
| 132 | } |
| 133 | |
| 134 | void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) { |
| 135 | enable_thread_checks_ = enable; |
| 136 | } |
| 137 | |
| 138 | bool URLRequestThrottlerManager::enable_thread_checks() const { |
| 139 | return enable_thread_checks_; |
| 140 | } |
| 141 | |
| 142 | void URLRequestThrottlerManager::set_net_log(NetLog* net_log) { |
| 143 | DCHECK(net_log); |
| 144 | net_log_ = BoundNetLog::Make(net_log, |
| 145 | NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING); |
| 146 | } |
| 147 | |
| 148 | NetLog* URLRequestThrottlerManager::net_log() const { |
| 149 | return net_log_.net_log(); |
| 150 | } |
| 151 | |
| 152 | void URLRequestThrottlerManager::OnIPAddressChanged() { |
| 153 | OnNetworkChange(); |
| 154 | } |
| 155 | |
| 156 | void URLRequestThrottlerManager::OnConnectionTypeChanged( |
| 157 | NetworkChangeNotifier::ConnectionType type) { |
| 158 | OnNetworkChange(); |
| 159 | } |
| 160 | |
| 161 | std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const { |
| 162 | if (!url.is_valid()) |
| 163 | return url.possibly_invalid_spec(); |
| 164 | |
| 165 | GURL id = url.ReplaceComponents(url_id_replacements_); |
Torne (Richard Coles) | 6e8cce6 | 2014-08-19 13:00:08 +0100 | [diff] [blame] | 166 | return base::StringToLowerASCII(id.spec()).c_str(); |
Torne (Richard Coles) | 5821806 | 2012-11-14 11:43:16 +0000 | [diff] [blame] | 167 | } |
| 168 | |
| 169 | void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() { |
| 170 | requests_since_last_gc_++; |
| 171 | if (requests_since_last_gc_ < kRequestsBetweenCollecting) |
| 172 | return; |
| 173 | requests_since_last_gc_ = 0; |
| 174 | |
| 175 | GarbageCollectEntries(); |
| 176 | } |
| 177 | |
| 178 | void URLRequestThrottlerManager::GarbageCollectEntries() { |
| 179 | UrlEntryMap::iterator i = url_entries_.begin(); |
| 180 | while (i != url_entries_.end()) { |
| 181 | if ((i->second)->IsEntryOutdated()) { |
| 182 | url_entries_.erase(i++); |
| 183 | } else { |
| 184 | ++i; |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | // In case something broke we want to make sure not to grow indefinitely. |
| 189 | while (url_entries_.size() > kMaximumNumberOfEntries) { |
| 190 | url_entries_.erase(url_entries_.begin()); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | void URLRequestThrottlerManager::OnNetworkChange() { |
| 195 | // Remove all entries. Any entries that in-flight requests have a reference |
| 196 | // to will live until those requests end, and these entries may be |
| 197 | // inconsistent with new entries for the same URLs, but since what we |
| 198 | // want is a clean slate for the new connection type, this is OK. |
| 199 | url_entries_.clear(); |
| 200 | requests_since_last_gc_ = 0; |
| 201 | } |
| 202 | |
| 203 | } // namespace net |