blob: c93dfacbe082b20253e4f58c9c4ae9c2d5d2ad04 [file] [log] [blame]
David Garcia Quintas3fb8f732016-06-15 22:53:08 -07001/*
2 *
3 * Copyright 2016, Google Inc.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above
13 * copyright notice, this list of conditions and the following disclaimer
14 * in the documentation and/or other materials provided with the
15 * distribution.
16 * * Neither the name of Google Inc. nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 *
32 */
33
David Garcia Quintasd9cee6f2016-08-01 17:42:47 -070034#include <cinttypes>
David Garcia Quintasaaba1312016-06-22 18:10:37 -070035#include <cstdarg>
David Garcia Quintasd9cee6f2016-08-01 17:42:47 -070036#include <cstdint>
David Garcia Quintasaaba1312016-06-22 18:10:37 -070037#include <cstring>
38#include <string>
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070039
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -070040#include <gtest/gtest.h>
41
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070042#include <grpc/grpc.h>
David Garcia Quintas601bb122016-08-18 15:03:59 -070043#include <grpc/impl/codegen/byte_buffer_reader.h>
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070044#include <grpc/support/alloc.h>
45#include <grpc/support/host_port.h>
46#include <grpc/support/log.h>
47#include <grpc/support/string_util.h>
48#include <grpc/support/sync.h>
49#include <grpc/support/thd.h>
50#include <grpc/support/time.h>
51
David Garcia Quintas8a81aa12016-08-22 15:06:49 -070052#include <grpc++/impl/codegen/config.h>
David Garcia Quintas8a81aa12016-08-22 15:06:49 -070053extern "C" {
Mark D. Rotheb35a1d2016-10-19 14:53:42 -070054#include "src/core/ext/client_channel/client_channel.h"
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070055#include "src/core/lib/channel/channel_stack.h"
David Garcia Quintas88006452016-08-24 06:53:23 -070056#include "src/core/lib/iomgr/sockaddr.h"
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070057#include "src/core/lib/support/string.h"
58#include "src/core/lib/support/tmpfile.h"
59#include "src/core/lib/surface/channel.h"
60#include "src/core/lib/surface/server.h"
61#include "test/core/end2end/cq_verifier.h"
Mark D. Rothda0ec822016-10-03 11:32:04 -070062#include "test/core/end2end/fake_resolver.h"
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070063#include "test/core/util/port.h"
64#include "test/core/util/test_config.h"
David Garcia Quintas55145c02016-06-21 14:51:54 -070065}
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070066
David Garcia Quintasaaba1312016-06-22 18:10:37 -070067#include "src/proto/grpc/lb/v1/load_balancer.pb.h"
68
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070069#define NUM_BACKENDS 4
David Garcia Quintas5b0e9462016-08-15 19:38:39 -070070#define PAYLOAD "hello you"
David Garcia Quintas3fb8f732016-06-15 22:53:08 -070071
David Garcia Quintasf9f856b2016-06-22 18:25:53 -070072// TODO(dgq): Other scenarios in need of testing:
David Garcia Quintasea11d162016-07-14 17:27:28 -070073// - Send an empty serverlist update and verify that the client request blocks
74// until a new serverlist with actual contents is available.
David Garcia Quintasf9f856b2016-06-22 18:25:53 -070075// - Send identical serverlist update
David Garcia Quintas35c2aba2016-09-13 15:28:09 -070076// - Send a serverlist with faulty ip:port addresses (port > 2^16, etc).
David Garcia Quintasf9f856b2016-06-22 18:25:53 -070077// - Test reception of invalid serverlist
78// - Test pinging
David Garcia Quintas7ec29132016-11-01 04:09:05 +010079// - Test against a non-LB server.
David Garcia Quintasa26c77b2016-07-18 12:57:09 -070080// - Random LB server closing the stream unexpectedly.
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -070081// - Test using DNS-resolvable names (localhost?)
David Garcia Quintas149f09d2016-11-17 20:43:10 -080082// - Test handling of creation of faulty RR instance by having the LB return a
83// serverlist with non-existent backends after having initially returned a
84// valid one.
David Garcia Quintas7ec29132016-11-01 04:09:05 +010085//
86// Findings from end to end testing to be covered here:
87// - Handling of LB servers restart, including reconnection after backing-off
88// retries.
89// - Destruction of load balanced channel (and therefore of grpclb instance)
90// while:
91// 1) the internal LB call is still active. This should work by virtue
92// of the weak reference the LB call holds. The call should be terminated as
93// part of the grpclb shutdown process.
94// 2) the retry timer is active. Again, the weak reference it holds should
95// prevent a premature call to \a glb_destroy.
96// - Restart of backend servers with no changes to serverlist. This exercises
97// the RR handover mechanism.
David Garcia Quintasf9f856b2016-06-22 18:25:53 -070098
David Garcia Quintasaaba1312016-06-22 18:10:37 -070099namespace grpc {
100namespace {
101
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700102typedef struct client_fixture {
103 grpc_channel *client;
104 char *server_uri;
105 grpc_completion_queue *cq;
106} client_fixture;
107
108typedef struct server_fixture {
109 grpc_server *server;
110 grpc_call *server_call;
111 grpc_completion_queue *cq;
112 char *servers_hostport;
113 int port;
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800114 const char *lb_token_prefix;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700115 gpr_thd_id tid;
116 int num_calls_serviced;
117} server_fixture;
118
119typedef struct test_fixture {
120 server_fixture lb_server;
121 server_fixture lb_backends[NUM_BACKENDS];
122 client_fixture client;
123 int lb_server_update_delay_ms;
124} test_fixture;
125
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700126static void *tag(intptr_t t) { return (void *)t; }
127
Craig Tillerd41a4a72016-10-26 16:16:06 -0700128static grpc_slice build_response_payload_slice(
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700129 const char *host, int *ports, size_t nports,
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800130 int64_t expiration_interval_secs, int32_t expiration_interval_nanos,
131 const char *token_prefix) {
David Garcia Quintasf9f856b2016-06-22 18:25:53 -0700132 // server_list {
133 // servers {
David Garcia Quintas8a81aa12016-08-22 15:06:49 -0700134 // ip_address: <in_addr/6 bytes of an IP>
135 // port: <16 bit uint>
David Garcia Quintasf9f856b2016-06-22 18:25:53 -0700136 // load_balance_token: "token..."
137 // }
138 // ...
139 // }
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700140 grpc::lb::v1::LoadBalanceResponse response;
141 auto *serverlist = response.mutable_server_list();
142
143 if (expiration_interval_secs > 0 || expiration_interval_nanos > 0) {
144 auto *expiration_interval = serverlist->mutable_expiration_interval();
145 if (expiration_interval_secs > 0) {
146 expiration_interval->set_seconds(expiration_interval_secs);
147 }
148 if (expiration_interval_nanos > 0) {
149 expiration_interval->set_nanos(expiration_interval_nanos);
150 }
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700151 }
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700152 for (size_t i = 0; i < nports; i++) {
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700153 auto *server = serverlist->add_servers();
David Garcia Quintas8a81aa12016-08-22 15:06:49 -0700154 // TODO(dgq): test ipv6
155 struct in_addr ip4;
156 GPR_ASSERT(inet_pton(AF_INET, host, &ip4) == 1);
157 server->set_ip_address(
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800158 string(reinterpret_cast<const char *>(&ip4), sizeof(ip4)));
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700159 server->set_port(ports[i]);
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800160 // Missing tokens are acceptable. Test that path.
161 if (strlen(token_prefix) > 0) {
David Garcia Quintas0133a1b2016-11-15 15:42:37 -0800162 string token_data = token_prefix + std::to_string(ports[i]);
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800163 server->set_load_balance_token(token_data);
164 }
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700165 }
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800166 const string &enc_resp = response.SerializeAsString();
Craig Tillerd41a4a72016-10-26 16:16:06 -0700167 return grpc_slice_from_copied_buffer(enc_resp.data(), enc_resp.size());
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700168}
169
170static void drain_cq(grpc_completion_queue *cq) {
171 grpc_event ev;
172 do {
David Garcia Quintas4166cb02016-07-29 14:33:15 -0700173 ev = grpc_completion_queue_next(cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5),
174 NULL);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700175 } while (ev.type != GRPC_QUEUE_SHUTDOWN);
176}
177
178static void sleep_ms(int delay_ms) {
179 gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
180 gpr_time_from_millis(delay_ms, GPR_TIMESPAN)));
181}
182
183static void start_lb_server(server_fixture *sf, int *ports, size_t nports,
184 int update_delay_ms) {
185 grpc_call *s;
186 cq_verifier *cqv = cq_verifier_create(sf->cq);
187 grpc_op ops[6];
188 grpc_op *op;
189 grpc_metadata_array request_metadata_recv;
190 grpc_call_details call_details;
191 grpc_call_error error;
192 int was_cancelled = 2;
193 grpc_byte_buffer *request_payload_recv;
194 grpc_byte_buffer *response_payload;
195
196 memset(ops, 0, sizeof(ops));
197 grpc_metadata_array_init(&request_metadata_recv);
198 grpc_call_details_init(&call_details);
199
200 error = grpc_server_request_call(sf->server, &s, &call_details,
201 &request_metadata_recv, sf->cq, sf->cq,
202 tag(200));
203 GPR_ASSERT(GRPC_CALL_OK == error);
204 gpr_log(GPR_INFO, "LB Server[%s] up", sf->servers_hostport);
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700205 CQ_EXPECT_COMPLETION(cqv, tag(200), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700206 cq_verify(cqv);
207 gpr_log(GPR_INFO, "LB Server[%s] after tag 200", sf->servers_hostport);
208
David Garcia Quintas601bb122016-08-18 15:03:59 -0700209 // make sure we've received the initial metadata from the grpclb request.
210 GPR_ASSERT(request_metadata_recv.count > 0);
211 GPR_ASSERT(request_metadata_recv.metadata != NULL);
212
213 // receive request for backends
214 op = ops;
215 op->op = GRPC_OP_RECV_MESSAGE;
216 op->data.recv_message = &request_payload_recv;
217 op->flags = 0;
218 op->reserved = NULL;
219 op++;
220 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(202), NULL);
221 GPR_ASSERT(GRPC_CALL_OK == error);
David Garcia Quintasb8245bd2016-09-01 18:29:46 +0200222 CQ_EXPECT_COMPLETION(cqv, tag(202), 1);
David Garcia Quintas601bb122016-08-18 15:03:59 -0700223 cq_verify(cqv);
224 gpr_log(GPR_INFO, "LB Server[%s] after RECV_MSG", sf->servers_hostport);
225
226 // validate initial request.
227 grpc_byte_buffer_reader bbr;
228 grpc_byte_buffer_reader_init(&bbr, request_payload_recv);
Craig Tillerd41a4a72016-10-26 16:16:06 -0700229 grpc_slice request_payload_slice = grpc_byte_buffer_reader_readall(&bbr);
David Garcia Quintas601bb122016-08-18 15:03:59 -0700230 grpc::lb::v1::LoadBalanceRequest request;
Craig Tiller618e67d2016-10-26 21:08:10 -0700231 request.ParseFromArray(GRPC_SLICE_START_PTR(request_payload_slice),
232 GRPC_SLICE_LENGTH(request_payload_slice));
David Garcia Quintas601bb122016-08-18 15:03:59 -0700233 GPR_ASSERT(request.has_initial_request());
David Garcia Quintas55ba14a2016-09-27 18:45:30 -0700234 GPR_ASSERT(request.initial_request().name() == sf->servers_hostport);
Craig Tillerd41a4a72016-10-26 16:16:06 -0700235 grpc_slice_unref(request_payload_slice);
David Garcia Quintas601bb122016-08-18 15:03:59 -0700236 grpc_byte_buffer_reader_destroy(&bbr);
237 grpc_byte_buffer_destroy(request_payload_recv);
238
Craig Tillerd41a4a72016-10-26 16:16:06 -0700239 grpc_slice response_payload_slice;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700240 op = ops;
241 op->op = GRPC_OP_SEND_INITIAL_METADATA;
242 op->data.send_initial_metadata.count = 0;
243 op->flags = 0;
244 op->reserved = NULL;
245 op++;
246 op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
247 op->data.recv_close_on_server.cancelled = &was_cancelled;
248 op->flags = 0;
249 op->reserved = NULL;
250 op++;
251 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(201), NULL);
252 GPR_ASSERT(GRPC_CALL_OK == error);
253 gpr_log(GPR_INFO, "LB Server[%s] after tag 201", sf->servers_hostport);
254
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700255 for (int i = 0; i < 2; i++) {
256 if (i == 0) {
257 // First half of the ports.
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800258 response_payload_slice = build_response_payload_slice(
259 "127.0.0.1", ports, nports / 2, -1, -1, sf->lb_token_prefix);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700260 } else {
261 // Second half of the ports.
262 sleep_ms(update_delay_ms);
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800263 response_payload_slice = build_response_payload_slice(
264 "127.0.0.1", ports + (nports / 2), (nports + 1) / 2 /* ceil */, -1,
265 -1, "" /* this half doesn't get to receive an LB token */);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700266 }
267
268 response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1);
269 op = ops;
270 op->op = GRPC_OP_SEND_MESSAGE;
271 op->data.send_message = response_payload;
272 op->flags = 0;
273 op->reserved = NULL;
274 op++;
275 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(203), NULL);
276 GPR_ASSERT(GRPC_CALL_OK == error);
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700277 CQ_EXPECT_COMPLETION(cqv, tag(203), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700278 cq_verify(cqv);
279 gpr_log(GPR_INFO, "LB Server[%s] after SEND_MESSAGE, iter %d",
280 sf->servers_hostport, i);
281
282 grpc_byte_buffer_destroy(response_payload);
Craig Tillerd41a4a72016-10-26 16:16:06 -0700283 grpc_slice_unref(response_payload_slice);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700284 }
285 gpr_log(GPR_INFO, "LB Server[%s] shutting down", sf->servers_hostport);
286
287 op = ops;
288 op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
289 op->data.send_status_from_server.trailing_metadata_count = 0;
290 op->data.send_status_from_server.status = GRPC_STATUS_OK;
Craig Tiller18348a32016-12-07 09:58:31 -0800291 grpc_slice status_details = grpc_slice_from_static_string("xyz");
292 op->data.send_status_from_server.status_details = &status_details;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700293 op->flags = 0;
294 op->reserved = NULL;
295 op++;
296 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(204), NULL);
297 GPR_ASSERT(GRPC_CALL_OK == error);
298
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700299 CQ_EXPECT_COMPLETION(cqv, tag(201), 1);
300 CQ_EXPECT_COMPLETION(cqv, tag(204), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700301 cq_verify(cqv);
302 gpr_log(GPR_INFO, "LB Server[%s] after tag 204. All done. LB server out",
303 sf->servers_hostport);
304
305 grpc_call_destroy(s);
306
307 cq_verifier_destroy(cqv);
308
309 grpc_metadata_array_destroy(&request_metadata_recv);
310 grpc_call_details_destroy(&call_details);
311}
312
313static void start_backend_server(server_fixture *sf) {
314 grpc_call *s;
315 cq_verifier *cqv;
316 grpc_op ops[6];
317 grpc_op *op;
318 grpc_metadata_array request_metadata_recv;
319 grpc_call_details call_details;
320 grpc_call_error error;
321 int was_cancelled;
322 grpc_byte_buffer *request_payload_recv;
323 grpc_byte_buffer *response_payload;
324 grpc_event ev;
325
326 while (true) {
327 memset(ops, 0, sizeof(ops));
328 cqv = cq_verifier_create(sf->cq);
329 was_cancelled = 2;
330 grpc_metadata_array_init(&request_metadata_recv);
331 grpc_call_details_init(&call_details);
332
333 error = grpc_server_request_call(sf->server, &s, &call_details,
334 &request_metadata_recv, sf->cq, sf->cq,
335 tag(100));
336 GPR_ASSERT(GRPC_CALL_OK == error);
337 gpr_log(GPR_INFO, "Server[%s] up", sf->servers_hostport);
David Garcia Quintas4166cb02016-07-29 14:33:15 -0700338 ev = grpc_completion_queue_next(sf->cq,
339 GRPC_TIMEOUT_SECONDS_TO_DEADLINE(60), NULL);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700340 if (!ev.success) {
341 gpr_log(GPR_INFO, "Server[%s] being torn down", sf->servers_hostport);
342 cq_verifier_destroy(cqv);
343 grpc_metadata_array_destroy(&request_metadata_recv);
344 grpc_call_details_destroy(&call_details);
345 return;
346 }
347 GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800348 const string expected_token =
Craig Tillerf658bf02016-12-08 14:11:47 -0800349 strlen(sf->lb_token_prefix) == 0 ? "" : sf->lb_token_prefix +
350 std::to_string(sf->port);
David Garcia Quintasa3654db2016-10-11 15:52:39 -0700351 GPR_ASSERT(contains_metadata(&request_metadata_recv, "lb-token",
David Garcia Quintas8a81aa12016-08-22 15:06:49 -0700352 expected_token.c_str()));
David Garcia Quintas5b0e9462016-08-15 19:38:39 -0700353
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700354 gpr_log(GPR_INFO, "Server[%s] after tag 100", sf->servers_hostport);
355
356 op = ops;
357 op->op = GRPC_OP_SEND_INITIAL_METADATA;
358 op->data.send_initial_metadata.count = 0;
359 op->flags = 0;
360 op->reserved = NULL;
361 op++;
362 op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
363 op->data.recv_close_on_server.cancelled = &was_cancelled;
364 op->flags = 0;
365 op->reserved = NULL;
366 op++;
367 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(101), NULL);
368 GPR_ASSERT(GRPC_CALL_OK == error);
369 gpr_log(GPR_INFO, "Server[%s] after tag 101", sf->servers_hostport);
370
371 bool exit = false;
Craig Tillerd41a4a72016-10-26 16:16:06 -0700372 grpc_slice response_payload_slice = grpc_slice_from_copied_string(PAYLOAD);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700373 while (!exit) {
374 op = ops;
375 op->op = GRPC_OP_RECV_MESSAGE;
376 op->data.recv_message = &request_payload_recv;
377 op->flags = 0;
378 op->reserved = NULL;
379 op++;
380 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL);
381 GPR_ASSERT(GRPC_CALL_OK == error);
David Garcia Quintas4166cb02016-07-29 14:33:15 -0700382 ev = grpc_completion_queue_next(
383 sf->cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700384 if (ev.type == GRPC_OP_COMPLETE && ev.success) {
385 GPR_ASSERT(ev.tag = tag(102));
386 if (request_payload_recv == NULL) {
387 exit = true;
388 gpr_log(GPR_INFO,
389 "Server[%s] recv \"close\" from client, exiting. Call #%d",
390 sf->servers_hostport, sf->num_calls_serviced);
391 }
392 } else {
393 gpr_log(GPR_INFO, "Server[%s] forced to shutdown. Call #%d",
394 sf->servers_hostport, sf->num_calls_serviced);
395 exit = true;
396 }
397 gpr_log(GPR_INFO, "Server[%s] after tag 102. Call #%d",
398 sf->servers_hostport, sf->num_calls_serviced);
399
400 if (!exit) {
401 response_payload =
402 grpc_raw_byte_buffer_create(&response_payload_slice, 1);
403 op = ops;
404 op->op = GRPC_OP_SEND_MESSAGE;
405 op->data.send_message = response_payload;
406 op->flags = 0;
407 op->reserved = NULL;
408 op++;
409 error =
410 grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(103), NULL);
411 GPR_ASSERT(GRPC_CALL_OK == error);
David Garcia Quintas4166cb02016-07-29 14:33:15 -0700412 ev = grpc_completion_queue_next(
413 sf->cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700414 if (ev.type == GRPC_OP_COMPLETE && ev.success) {
415 GPR_ASSERT(ev.tag = tag(103));
416 } else {
417 gpr_log(GPR_INFO, "Server[%s] forced to shutdown. Call #%d",
418 sf->servers_hostport, sf->num_calls_serviced);
419 exit = true;
420 }
421 gpr_log(GPR_INFO, "Server[%s] after tag 103. Call #%d",
422 sf->servers_hostport, sf->num_calls_serviced);
423 grpc_byte_buffer_destroy(response_payload);
424 }
425
426 grpc_byte_buffer_destroy(request_payload_recv);
427 }
428 ++sf->num_calls_serviced;
429
430 gpr_log(GPR_INFO, "Server[%s] OUT OF THE LOOP", sf->servers_hostport);
Craig Tillerd41a4a72016-10-26 16:16:06 -0700431 grpc_slice_unref(response_payload_slice);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700432
433 op = ops;
434 op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
435 op->data.send_status_from_server.trailing_metadata_count = 0;
436 op->data.send_status_from_server.status = GRPC_STATUS_OK;
Craig Tiller18348a32016-12-07 09:58:31 -0800437 grpc_slice status_details =
438 grpc_slice_from_static_string("Backend server out a-ok");
439 op->data.send_status_from_server.status_details = &status_details;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700440 op->flags = 0;
441 op->reserved = NULL;
442 op++;
443 error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(104), NULL);
444 GPR_ASSERT(GRPC_CALL_OK == error);
445
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700446 CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
447 CQ_EXPECT_COMPLETION(cqv, tag(104), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700448 cq_verify(cqv);
449 gpr_log(GPR_INFO, "Server[%s] DONE. After servicing %d calls",
David Garcia Quintas8782d1b2016-06-15 23:58:44 -0700450 sf->servers_hostport, sf->num_calls_serviced);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700451
452 grpc_call_destroy(s);
453 cq_verifier_destroy(cqv);
454 grpc_metadata_array_destroy(&request_metadata_recv);
455 grpc_call_details_destroy(&call_details);
456 }
457}
458
459static void perform_request(client_fixture *cf) {
460 grpc_call *c;
461 cq_verifier *cqv = cq_verifier_create(cf->cq);
462 grpc_op ops[6];
463 grpc_op *op;
464 grpc_metadata_array initial_metadata_recv;
465 grpc_metadata_array trailing_metadata_recv;
466 grpc_status_code status;
467 grpc_call_error error;
Craig Tiller18348a32016-12-07 09:58:31 -0800468 grpc_slice details;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700469 grpc_byte_buffer *request_payload;
470 grpc_byte_buffer *response_payload_recv;
471 int i;
472
473 memset(ops, 0, sizeof(ops));
Craig Tiller28b72422016-10-26 21:15:29 -0700474 grpc_slice request_payload_slice =
475 grpc_slice_from_copied_string("hello world");
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700476
Craig Tiller18348a32016-12-07 09:58:31 -0800477 grpc_slice host = grpc_slice_from_static_string("foo.test.google.fr:1234");
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700478 c = grpc_channel_create_call(cf->client, NULL, GRPC_PROPAGATE_DEFAULTS,
Craig Tiller18348a32016-12-07 09:58:31 -0800479 cf->cq, grpc_slice_from_static_string("/foo"),
480 &host, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5),
481 NULL);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700482 gpr_log(GPR_INFO, "Call 0x%" PRIxPTR " created", (intptr_t)c);
483 GPR_ASSERT(c);
484 char *peer;
485
486 grpc_metadata_array_init(&initial_metadata_recv);
487 grpc_metadata_array_init(&trailing_metadata_recv);
488
489 op = ops;
490 op->op = GRPC_OP_SEND_INITIAL_METADATA;
491 op->data.send_initial_metadata.count = 0;
492 op->flags = 0;
493 op->reserved = NULL;
494 op++;
495 op->op = GRPC_OP_RECV_INITIAL_METADATA;
496 op->data.recv_initial_metadata = &initial_metadata_recv;
497 op->flags = 0;
498 op->reserved = NULL;
499 op++;
500 op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
501 op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
502 op->data.recv_status_on_client.status = &status;
503 op->data.recv_status_on_client.status_details = &details;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700504 op->flags = 0;
505 op->reserved = NULL;
506 op++;
507 error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL);
508 GPR_ASSERT(GRPC_CALL_OK == error);
509
510 for (i = 0; i < 4; i++) {
511 request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1);
512
513 op = ops;
514 op->op = GRPC_OP_SEND_MESSAGE;
515 op->data.send_message = request_payload;
516 op->flags = 0;
517 op->reserved = NULL;
518 op++;
519 op->op = GRPC_OP_RECV_MESSAGE;
520 op->data.recv_message = &response_payload_recv;
521 op->flags = 0;
522 op->reserved = NULL;
523 op++;
524 error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(2), NULL);
525 GPR_ASSERT(GRPC_CALL_OK == error);
526
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700527 CQ_EXPECT_COMPLETION(cqv, tag(2), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700528 cq_verify(cqv);
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800529 gpr_log(GPR_INFO, "Client after sending msg %d / 4", i + 1);
David Garcia Quintas5b0e9462016-08-15 19:38:39 -0700530 GPR_ASSERT(byte_buffer_eq_string(response_payload_recv, PAYLOAD));
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700531
532 grpc_byte_buffer_destroy(request_payload);
533 grpc_byte_buffer_destroy(response_payload_recv);
534 }
535
Craig Tillerd41a4a72016-10-26 16:16:06 -0700536 grpc_slice_unref(request_payload_slice);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700537
538 op = ops;
539 op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
540 op->flags = 0;
541 op->reserved = NULL;
542 op++;
543 error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(3), NULL);
544 GPR_ASSERT(GRPC_CALL_OK == error);
545
Mark D. Roth7f9bba82016-08-25 08:35:42 -0700546 CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
547 CQ_EXPECT_COMPLETION(cqv, tag(3), 1);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700548 cq_verify(cqv);
549 peer = grpc_call_get_peer(c);
550 gpr_log(GPR_INFO, "Client DONE WITH SERVER %s ", peer);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700551
552 grpc_call_destroy(c);
553
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800554 cq_verify_empty_timeout(cqv, 1 /* seconds */);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700555 cq_verifier_destroy(cqv);
556
557 grpc_metadata_array_destroy(&initial_metadata_recv);
558 grpc_metadata_array_destroy(&trailing_metadata_recv);
Craig Tiller18348a32016-12-07 09:58:31 -0800559 grpc_slice_unref(details);
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800560 gpr_log(GPR_INFO, "Client call (peer %s) DESTROYED.", peer);
561 gpr_free(peer);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700562}
563
564static void setup_client(const char *server_hostport, client_fixture *cf) {
565 cf->cq = grpc_completion_queue_create(NULL);
566 cf->server_uri = gpr_strdup(server_hostport);
567 cf->client = grpc_insecure_channel_create(cf->server_uri, NULL, NULL);
568}
569
570static void teardown_client(client_fixture *cf) {
571 grpc_completion_queue_shutdown(cf->cq);
572 drain_cq(cf->cq);
573 grpc_completion_queue_destroy(cf->cq);
574 cf->cq = NULL;
575 grpc_channel_destroy(cf->client);
576 cf->client = NULL;
577 gpr_free(cf->server_uri);
578}
579
580static void setup_server(const char *host, server_fixture *sf) {
581 int assigned_port;
582
583 sf->cq = grpc_completion_queue_create(NULL);
David Garcia Quintas55145c02016-06-21 14:51:54 -0700584 const char *colon_idx = strchr(host, ':');
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700585 if (colon_idx) {
David Garcia Quintas55145c02016-06-21 14:51:54 -0700586 const char *port_str = colon_idx + 1;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700587 sf->port = atoi(port_str);
588 sf->servers_hostport = gpr_strdup(host);
589 } else {
590 sf->port = grpc_pick_unused_port_or_die();
591 gpr_join_host_port(&sf->servers_hostport, host, sf->port);
592 }
593
594 sf->server = grpc_server_create(NULL, NULL);
595 grpc_server_register_completion_queue(sf->server, sf->cq, NULL);
596 GPR_ASSERT((assigned_port = grpc_server_add_insecure_http2_port(
597 sf->server, sf->servers_hostport)) > 0);
598 GPR_ASSERT(sf->port == assigned_port);
599 grpc_server_start(sf->server);
600}
601
602static void teardown_server(server_fixture *sf) {
603 if (!sf->server) return;
604
605 gpr_log(GPR_INFO, "Server[%s] shutting down", sf->servers_hostport);
606 grpc_server_shutdown_and_notify(sf->server, sf->cq, tag(1000));
607 GPR_ASSERT(grpc_completion_queue_pluck(
608 sf->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL)
609 .type == GRPC_OP_COMPLETE);
610 grpc_server_destroy(sf->server);
611 gpr_thd_join(sf->tid);
612
613 sf->server = NULL;
614 grpc_completion_queue_shutdown(sf->cq);
615 drain_cq(sf->cq);
616 grpc_completion_queue_destroy(sf->cq);
617
618 gpr_log(GPR_INFO, "Server[%s] bye bye", sf->servers_hostport);
619 gpr_free(sf->servers_hostport);
620}
621
622static void fork_backend_server(void *arg) {
David Garcia Quintas55145c02016-06-21 14:51:54 -0700623 server_fixture *sf = static_cast<server_fixture *>(arg);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700624 start_backend_server(sf);
625}
626
627static void fork_lb_server(void *arg) {
David Garcia Quintas55145c02016-06-21 14:51:54 -0700628 test_fixture *tf = static_cast<test_fixture *>(arg);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700629 int ports[NUM_BACKENDS];
630 for (int i = 0; i < NUM_BACKENDS; i++) {
631 ports[i] = tf->lb_backends[i].port;
632 }
633 start_lb_server(&tf->lb_server, ports, NUM_BACKENDS,
634 tf->lb_server_update_delay_ms);
635}
636
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800637#define LB_TOKEN_PREFIX "token"
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700638static test_fixture setup_test_fixture(int lb_server_update_delay_ms) {
639 test_fixture tf;
640 memset(&tf, 0, sizeof(tf));
641 tf.lb_server_update_delay_ms = lb_server_update_delay_ms;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700642
643 gpr_thd_options options = gpr_thd_options_default();
644 gpr_thd_options_set_joinable(&options);
645
646 for (int i = 0; i < NUM_BACKENDS; ++i) {
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800647 // Only the first half of the servers expect an LB token.
648 if (i < NUM_BACKENDS / 2) {
649 tf.lb_backends[i].lb_token_prefix = LB_TOKEN_PREFIX;
650 } else {
651 tf.lb_backends[i].lb_token_prefix = "";
652 }
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700653 setup_server("127.0.0.1", &tf.lb_backends[i]);
654 gpr_thd_new(&tf.lb_backends[i].tid, fork_backend_server, &tf.lb_backends[i],
655 &options);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700656 }
657
David Garcia Quintas850cbaa2016-11-15 15:13:35 -0800658 tf.lb_server.lb_token_prefix = LB_TOKEN_PREFIX;
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700659 setup_server("127.0.0.1", &tf.lb_server);
660 gpr_thd_new(&tf.lb_server.tid, fork_lb_server, &tf.lb_server, &options);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700661
662 char *server_uri;
Mark D. Rothe127a392016-10-27 08:27:15 -0700663 // The grpclb LB policy will be automatically selected by virtue of
664 // the fact that the returned addresses are balancer addresses.
665 gpr_asprintf(&server_uri, "test:%s?lb_enabled=1",
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700666 tf.lb_server.servers_hostport);
667 setup_client(server_uri, &tf.client);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700668 gpr_free(server_uri);
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700669
670 return tf;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700671}
672
673static void teardown_test_fixture(test_fixture *tf) {
674 teardown_client(&tf->client);
675 for (int i = 0; i < NUM_BACKENDS; ++i) {
676 teardown_server(&tf->lb_backends[i]);
677 }
678 teardown_server(&tf->lb_server);
679}
680
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700681// The LB server will send two updates: batch 1 and batch 2. Each batch contains
682// two addresses, both of a valid and running backend server. Batch 1 is readily
683// available and provided as soon as the client establishes the streaming call.
684// Batch 2 is sent after a delay of \a lb_server_update_delay_ms milliseconds.
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700685static test_fixture test_update(int lb_server_update_delay_ms) {
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700686 gpr_log(GPR_INFO, "start %s(%d)", __func__, lb_server_update_delay_ms);
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700687 test_fixture tf = setup_test_fixture(lb_server_update_delay_ms);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700688 perform_request(
689 &tf.client); // "consumes" 1st backend server of 1st serverlist
690 perform_request(
691 &tf.client); // "consumes" 2nd backend server of 1st serverlist
692
693 perform_request(
694 &tf.client); // "consumes" 1st backend server of 2nd serverlist
695 perform_request(
696 &tf.client); // "consumes" 2nd backend server of 2nd serverlist
697
698 teardown_test_fixture(&tf);
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700699 gpr_log(GPR_INFO, "end %s(%d)", __func__, lb_server_update_delay_ms);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700700 return tf;
701}
702
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700703TEST(GrpclbTest, Updates) {
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700704 grpc::test_fixture tf_result;
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800705 // Clients take at least one second to complete a call (the last part of the
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700706 // call sleeps for 1 second while verifying the client's completion queue is
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800707 // empty), more if the system is under load. Therefore:
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700708 //
709 // If the LB server waits 800ms before sending an update, it will arrive
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800710 // before the first client request finishes, skipping the second server from
711 // batch 1. All subsequent picks will come from the second half of the
712 // backends, those coming in the LB update.
David Garcia Quintasaaba1312016-06-22 18:10:37 -0700713 tf_result = grpc::test_update(800);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700714 GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced == 1);
715 GPR_ASSERT(tf_result.lb_backends[1].num_calls_serviced == 0);
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800716 GPR_ASSERT(tf_result.lb_backends[2].num_calls_serviced +
717 tf_result.lb_backends[3].num_calls_serviced >
718 0);
719 int num_serviced_calls = 0;
720 for (int i = 0; i < 4; i++) {
721 num_serviced_calls += tf_result.lb_backends[i].num_calls_serviced;
722 }
723 GPR_ASSERT(num_serviced_calls == 4);
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700724
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800725 // If the LB server waits 2500ms, the update arrives after two calls and three
726 // picks. The third pick will be the 1st server of the 1st update (RR policy
727 // going around). The fourth and final pick will come from the second LB
728 // update. In any case, the total number of serviced calls must again be equal
729 // to four across all the backends.
David Garcia Quintase60ae9c2016-08-02 16:37:41 -0700730 tf_result = grpc::test_update(2500);
731 GPR_ASSERT(tf_result.lb_backends[0].num_calls_serviced >= 1);
David Garcia Quintas033c51e2016-11-30 00:49:29 -0800732 GPR_ASSERT(tf_result.lb_backends[1].num_calls_serviced == 1);
733 GPR_ASSERT(tf_result.lb_backends[2].num_calls_serviced +
734 tf_result.lb_backends[3].num_calls_serviced >
735 0);
736 num_serviced_calls = 0;
737 for (int i = 0; i < 4; i++) {
738 num_serviced_calls += tf_result.lb_backends[i].num_calls_serviced;
739 }
740 GPR_ASSERT(num_serviced_calls == 4);
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700741}
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700742
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700743TEST(GrpclbTest, InvalidAddressInServerlist) {}
744
745} // namespace
746} // namespace grpc
747
748int main(int argc, char **argv) {
749 ::testing::InitGoogleTest(&argc, argv);
750 grpc_test_init(argc, argv);
Mark D. Rothda0ec822016-10-03 11:32:04 -0700751 grpc_fake_resolver_init();
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700752 grpc_init();
753 const auto result = RUN_ALL_TESTS();
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700754 grpc_shutdown();
David Garcia Quintasf47d6fb2016-09-14 12:59:17 -0700755 return result;
David Garcia Quintas3fb8f732016-06-15 22:53:08 -0700756}