blob: a8cee33ae8d2eaa6187945026e307168c96d0b45 [file] [log] [blame]
George Zhang83e2ec72013-01-08 15:53:51 -08001/*
2 * VMware VMCI Driver
3 *
4 * Copyright (C) 2012 VMware, Inc. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation version 2 and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 */
15
16#include <linux/vmw_vmci_defs.h>
17#include <linux/vmw_vmci_api.h>
18#include <linux/completion.h>
19#include <linux/hash.h>
20#include <linux/kernel.h>
21#include <linux/list.h>
22#include <linux/module.h>
23#include <linux/sched.h>
24#include <linux/slab.h>
25
26#include "vmci_datagram.h"
27#include "vmci_doorbell.h"
28#include "vmci_resource.h"
29#include "vmci_driver.h"
30#include "vmci_route.h"
31
32
33#define VMCI_DOORBELL_INDEX_BITS 6
34#define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS)
35#define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
36
37/*
38 * DoorbellEntry describes the a doorbell notification handle allocated by the
39 * host.
40 */
41struct dbell_entry {
42 struct vmci_resource resource;
43 struct hlist_node node;
44 struct work_struct work;
45 vmci_callback notify_cb;
46 void *client_data;
47 u32 idx;
48 u32 priv_flags;
49 bool run_delayed;
50 atomic_t active; /* Only used by guest personality */
51};
52
53/* The VMCI index table keeps track of currently registered doorbells. */
54struct dbell_index_table {
55 spinlock_t lock; /* Index table lock */
56 struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
57};
58
59static struct dbell_index_table vmci_doorbell_it = {
60 .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
61};
62
63/*
64 * The max_notify_idx is one larger than the currently known bitmap index in
65 * use, and is used to determine how much of the bitmap needs to be scanned.
66 */
67static u32 max_notify_idx;
68
69/*
70 * The notify_idx_count is used for determining whether there are free entries
71 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
72 */
73static u32 notify_idx_count;
74
75/*
76 * The last_notify_idx_reserved is used to track the last index handed out - in
77 * the case where multiple handles share a notification index, we hand out
78 * indexes round robin based on last_notify_idx_reserved.
79 */
80static u32 last_notify_idx_reserved;
81
82/* This is a one entry cache used to by the index allocation. */
83static u32 last_notify_idx_released = PAGE_SIZE;
84
85
86/*
87 * Utility function that retrieves the privilege flags associated
88 * with a given doorbell handle. For guest endpoints, the
89 * privileges are determined by the context ID, but for host
90 * endpoints privileges are associated with the complete
91 * handle. Hypervisor endpoints are not yet supported.
92 */
93int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
94{
95 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
96 return VMCI_ERROR_INVALID_ARGS;
97
98 if (handle.context == VMCI_HOST_CONTEXT_ID) {
99 struct dbell_entry *entry;
100 struct vmci_resource *resource;
101
102 resource = vmci_resource_by_handle(handle,
103 VMCI_RESOURCE_TYPE_DOORBELL);
104 if (!resource)
105 return VMCI_ERROR_NOT_FOUND;
106
107 entry = container_of(resource, struct dbell_entry, resource);
108 *priv_flags = entry->priv_flags;
109 vmci_resource_put(resource);
110 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
111 /*
112 * Hypervisor endpoints for notifications are not
113 * supported (yet).
114 */
115 return VMCI_ERROR_INVALID_ARGS;
116 } else {
117 *priv_flags = vmci_context_get_priv_flags(handle.context);
118 }
119
120 return VMCI_SUCCESS;
121}
122
123/*
124 * Find doorbell entry by bitmap index.
125 */
126static struct dbell_entry *dbell_index_table_find(u32 idx)
127{
128 u32 bucket = VMCI_DOORBELL_HASH(idx);
129 struct dbell_entry *dbell;
George Zhang83e2ec72013-01-08 15:53:51 -0800130
Sasha Levinb67bfe02013-02-27 17:06:00 -0800131 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
George Zhang83e2ec72013-01-08 15:53:51 -0800132 node) {
133 if (idx == dbell->idx)
134 return dbell;
135 }
136
137 return NULL;
138}
139
140/*
141 * Add the given entry to the index table. This willi take a reference to the
142 * entry's resource so that the entry is not deleted before it is removed from
143 * the * table.
144 */
145static void dbell_index_table_add(struct dbell_entry *entry)
146{
147 u32 bucket;
148 u32 new_notify_idx;
149
150 vmci_resource_get(&entry->resource);
151
152 spin_lock_bh(&vmci_doorbell_it.lock);
153
154 /*
155 * Below we try to allocate an index in the notification
156 * bitmap with "not too much" sharing between resources. If we
157 * use less that the full bitmap, we either add to the end if
158 * there are no unused flags within the currently used area,
159 * or we search for unused ones. If we use the full bitmap, we
160 * allocate the index round robin.
161 */
162 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
163 if (last_notify_idx_released < max_notify_idx &&
164 !dbell_index_table_find(last_notify_idx_released)) {
165 new_notify_idx = last_notify_idx_released;
166 last_notify_idx_released = PAGE_SIZE;
167 } else {
168 bool reused = false;
169 new_notify_idx = last_notify_idx_reserved;
170 if (notify_idx_count + 1 < max_notify_idx) {
171 do {
172 if (!dbell_index_table_find
173 (new_notify_idx)) {
174 reused = true;
175 break;
176 }
177 new_notify_idx = (new_notify_idx + 1) %
178 max_notify_idx;
179 } while (new_notify_idx !=
180 last_notify_idx_released);
181 }
182 if (!reused) {
183 new_notify_idx = max_notify_idx;
184 max_notify_idx++;
185 }
186 }
187 } else {
188 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
189 }
190
191 last_notify_idx_reserved = new_notify_idx;
192 notify_idx_count++;
193
194 entry->idx = new_notify_idx;
195 bucket = VMCI_DOORBELL_HASH(entry->idx);
196 hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
197
198 spin_unlock_bh(&vmci_doorbell_it.lock);
199}
200
201/*
202 * Remove the given entry from the index table. This will release() the
203 * entry's resource.
204 */
205static void dbell_index_table_remove(struct dbell_entry *entry)
206{
207 spin_lock_bh(&vmci_doorbell_it.lock);
208
209 hlist_del_init(&entry->node);
210
211 notify_idx_count--;
212 if (entry->idx == max_notify_idx - 1) {
213 /*
214 * If we delete an entry with the maximum known
215 * notification index, we take the opportunity to
216 * prune the current max. As there might be other
217 * unused indices immediately below, we lower the
218 * maximum until we hit an index in use.
219 */
220 while (max_notify_idx > 0 &&
221 !dbell_index_table_find(max_notify_idx - 1))
222 max_notify_idx--;
223 }
224
225 last_notify_idx_released = entry->idx;
226
227 spin_unlock_bh(&vmci_doorbell_it.lock);
228
229 vmci_resource_put(&entry->resource);
230}
231
232/*
233 * Creates a link between the given doorbell handle and the given
234 * index in the bitmap in the device backend. A notification state
235 * is created in hypervisor.
236 */
237static int dbell_link(struct vmci_handle handle, u32 notify_idx)
238{
239 struct vmci_doorbell_link_msg link_msg;
240
241 link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
242 VMCI_DOORBELL_LINK);
243 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
244 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
245 link_msg.handle = handle;
246 link_msg.notify_idx = notify_idx;
247
248 return vmci_send_datagram(&link_msg.hdr);
249}
250
251/*
252 * Unlinks the given doorbell handle from an index in the bitmap in
253 * the device backend. The notification state is destroyed in hypervisor.
254 */
255static int dbell_unlink(struct vmci_handle handle)
256{
257 struct vmci_doorbell_unlink_msg unlink_msg;
258
259 unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
260 VMCI_DOORBELL_UNLINK);
261 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
262 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
263 unlink_msg.handle = handle;
264
265 return vmci_send_datagram(&unlink_msg.hdr);
266}
267
268/*
269 * Notify another guest or the host. We send a datagram down to the
270 * host via the hypervisor with the notification info.
271 */
272static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
273{
274 struct vmci_doorbell_notify_msg notify_msg;
275
276 notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
277 VMCI_DOORBELL_NOTIFY);
278 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
279 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
280 notify_msg.handle = handle;
281
282 return vmci_send_datagram(&notify_msg.hdr);
283}
284
285/*
286 * Calls the specified callback in a delayed context.
287 */
288static void dbell_delayed_dispatch(struct work_struct *work)
289{
290 struct dbell_entry *entry = container_of(work,
291 struct dbell_entry, work);
292
293 entry->notify_cb(entry->client_data);
294 vmci_resource_put(&entry->resource);
295}
296
297/*
298 * Dispatches a doorbell notification to the host context.
299 */
300int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
301{
302 struct dbell_entry *entry;
303 struct vmci_resource *resource;
304
305 if (vmci_handle_is_invalid(handle)) {
306 pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
307 handle.context, handle.resource);
308 return VMCI_ERROR_INVALID_ARGS;
309 }
310
311 resource = vmci_resource_by_handle(handle,
312 VMCI_RESOURCE_TYPE_DOORBELL);
313 if (!resource) {
314 pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
315 handle.context, handle.resource);
316 return VMCI_ERROR_NOT_FOUND;
317 }
318
319 entry = container_of(resource, struct dbell_entry, resource);
320 if (entry->run_delayed) {
321 schedule_work(&entry->work);
322 } else {
323 entry->notify_cb(entry->client_data);
324 vmci_resource_put(resource);
325 }
326
327 return VMCI_SUCCESS;
328}
329
330/*
331 * Register the notification bitmap with the host.
332 */
333bool vmci_dbell_register_notification_bitmap(u32 bitmap_ppn)
334{
335 int result;
336 struct vmci_notify_bm_set_msg bitmap_set_msg;
337
338 bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
339 VMCI_SET_NOTIFY_BITMAP);
340 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
341 bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
342 VMCI_DG_HEADERSIZE;
343 bitmap_set_msg.bitmap_ppn = bitmap_ppn;
344
345 result = vmci_send_datagram(&bitmap_set_msg.hdr);
346 if (result != VMCI_SUCCESS) {
347 pr_devel("Failed to register (PPN=%u) as notification bitmap (error=%d)\n",
348 bitmap_ppn, result);
349 return false;
350 }
351 return true;
352}
353
354/*
355 * Executes or schedules the handlers for a given notify index.
356 */
357static void dbell_fire_entries(u32 notify_idx)
358{
359 u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
360 struct dbell_entry *dbell;
George Zhang83e2ec72013-01-08 15:53:51 -0800361
362 spin_lock_bh(&vmci_doorbell_it.lock);
363
Sasha Levinb67bfe02013-02-27 17:06:00 -0800364 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
George Zhang83e2ec72013-01-08 15:53:51 -0800365 if (dbell->idx == notify_idx &&
366 atomic_read(&dbell->active) == 1) {
367 if (dbell->run_delayed) {
368 vmci_resource_get(&dbell->resource);
369 schedule_work(&dbell->work);
370 } else {
371 dbell->notify_cb(dbell->client_data);
372 }
373 }
374 }
375
376 spin_unlock_bh(&vmci_doorbell_it.lock);
377}
378
379/*
380 * Scans the notification bitmap, collects pending notifications,
381 * resets the bitmap and invokes appropriate callbacks.
382 */
383void vmci_dbell_scan_notification_entries(u8 *bitmap)
384{
385 u32 idx;
386
387 for (idx = 0; idx < max_notify_idx; idx++) {
388 if (bitmap[idx] & 0x1) {
389 bitmap[idx] &= ~1;
390 dbell_fire_entries(idx);
391 }
392 }
393}
394
395/*
396 * vmci_doorbell_create() - Creates a doorbell
397 * @handle: A handle used to track the resource. Can be invalid.
398 * @flags: Flag that determines context of callback.
399 * @priv_flags: Privileges flags.
400 * @notify_cb: The callback to be ivoked when the doorbell fires.
401 * @client_data: A parameter to be passed to the callback.
402 *
403 * Creates a doorbell with the given callback. If the handle is
404 * VMCI_INVALID_HANDLE, a free handle will be assigned, if
405 * possible. The callback can be run immediately (potentially with
406 * locks held - the default) or delayed (in a kernel thread) by
407 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
408 * is selected, a given callback may not be run if the kernel is
409 * unable to allocate memory for the delayed execution (highly
410 * unlikely).
411 */
412int vmci_doorbell_create(struct vmci_handle *handle,
413 u32 flags,
414 u32 priv_flags,
415 vmci_callback notify_cb, void *client_data)
416{
417 struct dbell_entry *entry;
418 struct vmci_handle new_handle;
419 int result;
420
421 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
422 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
423 return VMCI_ERROR_INVALID_ARGS;
424
425 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
426 if (entry == NULL) {
427 pr_warn("Failed allocating memory for datagram entry\n");
428 return VMCI_ERROR_NO_MEM;
429 }
430
431 if (vmci_handle_is_invalid(*handle)) {
432 u32 context_id = vmci_get_context_id();
433
434 /* Let resource code allocate a free ID for us */
435 new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
436 } else {
437 bool valid_context = false;
438
439 /*
440 * Validate the handle. We must do both of the checks below
441 * because we can be acting as both a host and a guest at the
442 * same time. We always allow the host context ID, since the
443 * host functionality is in practice always there with the
444 * unified driver.
445 */
446 if (handle->context == VMCI_HOST_CONTEXT_ID ||
447 (vmci_guest_code_active() &&
448 vmci_get_context_id() == handle->context)) {
449 valid_context = true;
450 }
451
452 if (!valid_context || handle->resource == VMCI_INVALID_ID) {
453 pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
454 handle->context, handle->resource);
455 result = VMCI_ERROR_INVALID_ARGS;
456 goto free_mem;
457 }
458
459 new_handle = *handle;
460 }
461
462 entry->idx = 0;
463 INIT_HLIST_NODE(&entry->node);
464 entry->priv_flags = priv_flags;
465 INIT_WORK(&entry->work, dbell_delayed_dispatch);
466 entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
467 entry->notify_cb = notify_cb;
468 entry->client_data = client_data;
469 atomic_set(&entry->active, 0);
470
471 result = vmci_resource_add(&entry->resource,
472 VMCI_RESOURCE_TYPE_DOORBELL,
473 new_handle);
474 if (result != VMCI_SUCCESS) {
475 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
476 new_handle.context, new_handle.resource, result);
477 goto free_mem;
478 }
479
480 new_handle = vmci_resource_handle(&entry->resource);
481 if (vmci_guest_code_active()) {
482 dbell_index_table_add(entry);
483 result = dbell_link(new_handle, entry->idx);
484 if (VMCI_SUCCESS != result)
485 goto destroy_resource;
486
487 atomic_set(&entry->active, 1);
488 }
489
490 *handle = new_handle;
491
492 return result;
493
494 destroy_resource:
495 dbell_index_table_remove(entry);
496 vmci_resource_remove(&entry->resource);
497 free_mem:
498 kfree(entry);
499 return result;
500}
501EXPORT_SYMBOL_GPL(vmci_doorbell_create);
502
503/*
504 * vmci_doorbell_destroy() - Destroy a doorbell.
505 * @handle: The handle tracking the resource.
506 *
507 * Destroys a doorbell previously created with vmcii_doorbell_create. This
508 * operation may block waiting for a callback to finish.
509 */
510int vmci_doorbell_destroy(struct vmci_handle handle)
511{
512 struct dbell_entry *entry;
513 struct vmci_resource *resource;
514
515 if (vmci_handle_is_invalid(handle))
516 return VMCI_ERROR_INVALID_ARGS;
517
518 resource = vmci_resource_by_handle(handle,
519 VMCI_RESOURCE_TYPE_DOORBELL);
520 if (!resource) {
521 pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
522 handle.context, handle.resource);
523 return VMCI_ERROR_NOT_FOUND;
524 }
525
526 entry = container_of(resource, struct dbell_entry, resource);
527
528 if (vmci_guest_code_active()) {
529 int result;
530
531 dbell_index_table_remove(entry);
532
533 result = dbell_unlink(handle);
534 if (VMCI_SUCCESS != result) {
535
536 /*
537 * The only reason this should fail would be
538 * an inconsistency between guest and
539 * hypervisor state, where the guest believes
540 * it has an active registration whereas the
541 * hypervisor doesn't. One case where this may
542 * happen is if a doorbell is unregistered
543 * following a hibernation at a time where the
544 * doorbell state hasn't been restored on the
545 * hypervisor side yet. Since the handle has
546 * now been removed in the guest, we just
547 * print a warning and return success.
548 */
549 pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
550 handle.context, handle.resource, result);
551 }
552 }
553
554 /*
555 * Now remove the resource from the table. It might still be in use
556 * after this, in a callback or still on the delayed work queue.
557 */
558 vmci_resource_put(&entry->resource);
559 vmci_resource_remove(&entry->resource);
560
561 kfree(entry);
562
563 return VMCI_SUCCESS;
564}
565EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
566
567/*
568 * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
569 * @dst: The handlle identifying the doorbell resource
570 * @priv_flags: Priviledge flags.
571 *
572 * Generates a notification on the doorbell identified by the
573 * handle. For host side generation of notifications, the caller
574 * can specify what the privilege of the calling side is.
575 */
576int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
577{
578 int retval;
579 enum vmci_route route;
580 struct vmci_handle src;
581
582 if (vmci_handle_is_invalid(dst) ||
583 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
584 return VMCI_ERROR_INVALID_ARGS;
585
586 src = VMCI_INVALID_HANDLE;
587 retval = vmci_route(&src, &dst, false, &route);
588 if (retval < VMCI_SUCCESS)
589 return retval;
590
591 if (VMCI_ROUTE_AS_HOST == route)
592 return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
593 dst, priv_flags);
594
595 if (VMCI_ROUTE_AS_GUEST == route)
596 return dbell_notify_as_guest(dst, priv_flags);
597
598 pr_warn("Unknown route (%d) for doorbell\n", route);
599 return VMCI_ERROR_DST_UNREACHABLE;
600}
601EXPORT_SYMBOL_GPL(vmci_doorbell_notify);