lapic.c 46.5 KB
Newer Older
Eddie Dong's avatar
Eddie Dong committed
1
2
3
4
5
6
7

/*
 * Local APIC virtualization
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2007 Novell
 * Copyright (C) 2007 Intel
8
 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
Eddie Dong's avatar
Eddie Dong committed
9
10
11
12
13
14
15
16
17
18
19
20
 *
 * Authors:
 *   Dor Laor <dor.laor@qumranet.com>
 *   Gregory Haskins <ghaskins@novell.com>
 *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
 *
 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 */

21
#include <linux/kvm_host.h>
Eddie Dong's avatar
Eddie Dong committed
22
23
24
25
26
27
28
#include <linux/kvm.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/smp.h>
#include <linux/hrtimer.h>
#include <linux/io.h>
#include <linux/module.h>
Roman Zippel's avatar
Roman Zippel committed
29
#include <linux/math64.h>
30
#include <linux/slab.h>
Eddie Dong's avatar
Eddie Dong committed
31
32
33
34
35
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/page.h>
#include <asm/current.h>
#include <asm/apicdef.h>
Arun Sharma's avatar
Arun Sharma committed
36
#include <linux/atomic.h>
37
#include <linux/jump_label.h>
38
#include "kvm_cache_regs.h"
Eddie Dong's avatar
Eddie Dong committed
39
#include "irq.h"
40
#include "trace.h"
41
#include "x86.h"
Avi Kivity's avatar
Avi Kivity committed
42
#include "cpuid.h"
Eddie Dong's avatar
Eddie Dong committed
43

44
45
46
47
48
49
#ifndef CONFIG_X86_64
#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
#else
#define mod_64(x, y) ((x) % (y))
#endif

Eddie Dong's avatar
Eddie Dong committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#define PRId64 "d"
#define PRIx64 "llx"
#define PRIu64 "u"
#define PRIo64 "o"

#define APIC_BUS_CYCLE_NS 1

/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
#define apic_debug(fmt, arg...)

#define APIC_LVT_NUM			6
/* 14 is the version for Xeon and Pentium 8.4.8*/
#define APIC_VERSION			(0x14UL | ((APIC_LVT_NUM - 1) << 16))
#define LAPIC_MMIO_LENGTH		(1 << 12)
/* followed define is not in apicdef.h */
#define APIC_SHORT_MASK			0xc0000
#define APIC_DEST_NOSHORT		0x0
#define APIC_DEST_MASK			0x800
#define MAX_APIC_VECTOR			256
69
#define APIC_VECTORS_PER_REG		32
Eddie Dong's avatar
Eddie Dong committed
70
71
72

#define VEC_POS(v) ((v) & (32 - 1))
#define REG_POS(v) (((v) >> 5) << 4)
73

74
75
76
static unsigned int min_timer_period_us = 500;
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

Eddie Dong's avatar
Eddie Dong committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
{
	*((u32 *) (apic->regs + reg_off)) = val;
}

static inline int apic_test_and_set_vector(int vec, void *bitmap)
{
	return test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int apic_test_and_clear_vector(int vec, void *bitmap)
{
	return test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

92
93
94
95
96
static inline int apic_test_vector(int vec, void *bitmap)
{
	return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

97
98
99
100
101
102
103
104
bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	return apic_test_vector(vector, apic->regs + APIC_ISR) ||
		apic_test_vector(vector, apic->regs + APIC_IRR);
}

Eddie Dong's avatar
Eddie Dong committed
105
106
107
108
109
110
111
112
113
114
static inline void apic_set_vector(int vec, void *bitmap)
{
	set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline void apic_clear_vector(int vec, void *bitmap)
{
	clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
115
116
117
118
119
120
121
122
123
124
static inline int __apic_test_and_set_vector(int vec, void *bitmap)
{
	return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
{
	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

125
struct static_key_deferred apic_hw_disabled __read_mostly;
126
127
128
129
struct static_key_deferred apic_sw_disabled __read_mostly;

static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
{
130
	if ((kvm_apic_get_reg(apic, APIC_SPIV) ^ val) & APIC_SPIV_APIC_ENABLED) {
131
132
133
134
135
136
137
138
		if (val & APIC_SPIV_APIC_ENABLED)
			static_key_slow_dec_deferred(&apic_sw_disabled);
		else
			static_key_slow_inc(&apic_sw_disabled.key);
	}
	apic_set_reg(apic, APIC_SPIV, val);
}

Eddie Dong's avatar
Eddie Dong committed
139
140
static inline int apic_enabled(struct kvm_lapic *apic)
{
141
	return kvm_apic_sw_enabled(apic) &&	kvm_apic_hw_enabled(apic);
142
143
}

Eddie Dong's avatar
Eddie Dong committed
144
145
146
147
148
149
150
151
152
#define LVT_MASK	\
	(APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)

#define LINT_MASK	\
	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)

static inline int kvm_apic_id(struct kvm_lapic *apic)
{
153
	return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
Eddie Dong's avatar
Eddie Dong committed
154
155
}

156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
static void recalculate_apic_map(struct kvm *kvm)
{
	struct kvm_apic_map *new, *old = NULL;
	struct kvm_vcpu *vcpu;
	int i;

	new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL);

	mutex_lock(&kvm->arch.apic_map_lock);

	if (!new)
		goto out;

	new->ldr_bits = 8;
	/* flat mode is default */
	new->cid_shift = 8;
	new->cid_mask = 0;
	new->lid_mask = 0xff;

	kvm_for_each_vcpu(i, vcpu, kvm) {
		struct kvm_lapic *apic = vcpu->arch.apic;
		u16 cid, lid;
		u32 ldr;

		if (!kvm_apic_present(vcpu))
			continue;

		/*
		 * All APICs have to be configured in the same mode by an OS.
		 * We take advatage of this while building logical id loockup
		 * table. After reset APICs are in xapic/flat mode, so if we
		 * find apic with different setting we assume this is the mode
		 * OS wants all apics to be in; build lookup table accordingly.
		 */
		if (apic_x2apic_mode(apic)) {
			new->ldr_bits = 32;
			new->cid_shift = 16;
			new->cid_mask = new->lid_mask = 0xffff;
		} else if (kvm_apic_sw_enabled(apic) &&
				!new->cid_mask /* flat mode */ &&
				kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) {
			new->cid_shift = 4;
			new->cid_mask = 0xf;
			new->lid_mask = 0xf;
		}

		new->phys_map[kvm_apic_id(apic)] = apic;

		ldr = kvm_apic_get_reg(apic, APIC_LDR);
		cid = apic_cluster_id(new, ldr);
		lid = apic_logical_id(new, ldr);

		if (lid)
			new->logical_map[cid][ffs(lid) - 1] = apic;
	}
out:
	old = rcu_dereference_protected(kvm->arch.apic_map,
			lockdep_is_held(&kvm->arch.apic_map_lock));
	rcu_assign_pointer(kvm->arch.apic_map, new);
	mutex_unlock(&kvm->arch.apic_map_lock);

	if (old)
		kfree_rcu(old, rcu);
219

220
	kvm_vcpu_request_scan_ioapic(kvm);
221
222
223
224
225
226
227
228
229
230
231
232
233
234
}

static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id)
{
	apic_set_reg(apic, APIC_ID, id << 24);
	recalculate_apic_map(apic->vcpu->kvm);
}

static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
{
	apic_set_reg(apic, APIC_LDR, id);
	recalculate_apic_map(apic->vcpu->kvm);
}

Eddie Dong's avatar
Eddie Dong committed
235
236
static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
{
237
	return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
Eddie Dong's avatar
Eddie Dong committed
238
239
240
241
}

static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
{
242
	return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
Eddie Dong's avatar
Eddie Dong committed
243
244
}

245
246
static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
{
247
	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
248
249
250
		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT);
}

Eddie Dong's avatar
Eddie Dong committed
251
252
static inline int apic_lvtt_period(struct kvm_lapic *apic)
{
253
	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
254
255
256
257
258
		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC);
}

static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
{
259
	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
260
261
		apic->lapic_timer.timer_mode_mask) ==
			APIC_LVT_TIMER_TSCDEADLINE);
Eddie Dong's avatar
Eddie Dong committed
262
263
}

264
265
266
267
268
static inline int apic_lvt_nmi_mode(u32 lvt_val)
{
	return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
}

269
270
271
272
273
274
void kvm_apic_set_version(struct kvm_vcpu *vcpu)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	struct kvm_cpuid_entry2 *feat;
	u32 v = APIC_VERSION;

275
	if (!kvm_vcpu_has_lapic(vcpu))
276
277
278
279
280
281
282
283
		return;

	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
	if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
		v |= APIC_LVR_DIRECTED_EOI;
	apic_set_reg(apic, APIC_LVR, v);
}

Mathias Krause's avatar
Mathias Krause committed
284
static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
285
	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
Eddie Dong's avatar
Eddie Dong committed
286
287
288
289
290
291
292
293
	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
	LINT_MASK, LINT_MASK,	/* LVT0-1 */
	LVT_MASK		/* LVTERR */
};

static int find_highest_vector(void *bitmap)
{
294
295
	int vec;
	u32 *reg;
Eddie Dong's avatar
Eddie Dong committed
296

297
298
299
300
301
302
	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
	     vec >= 0; vec -= APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		if (*reg)
			return fls(*reg) - 1 + vec;
	}
Eddie Dong's avatar
Eddie Dong committed
303

304
	return -1;
Eddie Dong's avatar
Eddie Dong committed
305
306
}

Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
307
308
static u8 count_vectors(void *bitmap)
{
309
310
	int vec;
	u32 *reg;
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
311
	u8 count = 0;
312
313
314
315
316
317

	for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
		reg = bitmap + REG_POS(vec);
		count += hweight32(*reg);
	}

Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
318
319
320
	return count;
}

321
322
323
324
325
326
327
328
329
330
331
332
333
void kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
{
	u32 i, pir_val;
	struct kvm_lapic *apic = vcpu->arch.apic;

	for (i = 0; i <= 7; i++) {
		pir_val = xchg(&pir[i], 0);
		if (pir_val)
			*((u32 *)(apic->regs + APIC_IRR + i * 0x10)) |= pir_val;
	}
}
EXPORT_SYMBOL_GPL(kvm_apic_update_irr);

Eddie Dong's avatar
Eddie Dong committed
334
335
static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic)
{
336
	apic->irr_pending = true;
Eddie Dong's avatar
Eddie Dong committed
337
338
339
	return apic_test_and_set_vector(vec, apic->regs + APIC_IRR);
}

340
static inline int apic_search_irr(struct kvm_lapic *apic)
Eddie Dong's avatar
Eddie Dong committed
341
{
342
	return find_highest_vector(apic->regs + APIC_IRR);
Eddie Dong's avatar
Eddie Dong committed
343
344
345
346
347
348
}

static inline int apic_find_highest_irr(struct kvm_lapic *apic)
{
	int result;

349
350
351
352
	/*
	 * Note that irr_pending is just a hint. It will be always
	 * true with virtual interrupt delivery enabled.
	 */
353
354
355
	if (!apic->irr_pending)
		return -1;

356
	kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
357
	result = apic_search_irr(apic);
Eddie Dong's avatar
Eddie Dong committed
358
359
360
361
362
	ASSERT(result == -1 || result >= 16);

	return result;
}

363
364
365
366
367
368
369
370
static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
{
	apic->irr_pending = false;
	apic_clear_vector(vec, apic->regs + APIC_IRR);
	if (apic_search_irr(apic) != -1)
		apic->irr_pending = true;
}

Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
{
	if (!__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
		++apic->isr_count;
	BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
	/*
	 * ISR (in service register) bit is set when injecting an interrupt.
	 * The highest vector is injected. Thus the latest bit set matches
	 * the highest bit in ISR.
	 */
	apic->highest_isr_cache = vec;
}

static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
{
	if (__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
		--apic->isr_count;
	BUG_ON(apic->isr_count < 0);
	apic->highest_isr_cache = -1;
}

392
393
394
395
int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
{
	int highest_irr;

396
397
398
399
400
	/* This may race with setting of irr in __apic_accept_irq() and
	 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
	 * will cause vmexit immediately and the value will be recalculated
	 * on the next vmentry.
	 */
401
	if (!kvm_vcpu_has_lapic(vcpu))
402
		return 0;
403
	highest_irr = apic_find_highest_irr(vcpu->arch.apic);
404
405
406
407

	return highest_irr;
}

408
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
409
410
			     int vector, int level, int trig_mode,
			     unsigned long *dest_map);
411

412
413
int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
		unsigned long *dest_map)
Eddie Dong's avatar
Eddie Dong committed
414
{
415
	struct kvm_lapic *apic = vcpu->arch.apic;
416

417
	return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
418
			irq->level, irq->trig_mode, dest_map);
Eddie Dong's avatar
Eddie Dong committed
419
420
}

421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
{

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
				      sizeof(val));
}

static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
				      sizeof(*val));
}

static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
}

static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
{
	u8 val;
	if (pv_eoi_get_user(vcpu, &val) < 0)
		apic_debug("Can't read EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
	return val & 0x1;
}

static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
		apic_debug("Can't set EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
		return;
	}
	__set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
{
	if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
		apic_debug("Can't clear EOI MSR value: 0x%llx\n",
			   (unsigned long long)vcpi->arch.pv_eoi.msr_val);
		return;
	}
	__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
}

Eddie Dong's avatar
Eddie Dong committed
469
470
471
static inline int apic_find_highest_isr(struct kvm_lapic *apic)
{
	int result;
472
473

	/* Note that isr_count is always 1 with vid enabled */
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
474
475
476
477
	if (!apic->isr_count)
		return -1;
	if (likely(apic->highest_isr_cache != -1))
		return apic->highest_isr_cache;
Eddie Dong's avatar
Eddie Dong committed
478
479
480
481
482
483
484

	result = find_highest_vector(apic->regs + APIC_ISR);
	ASSERT(result == -1 || result >= 16);

	return result;
}

485
486
487
488
489
490
491
492
493
void kvm_apic_update_tmr(struct kvm_vcpu *vcpu, u32 *tmr)
{
	struct kvm_lapic *apic = vcpu->arch.apic;
	int i;

	for (i = 0; i < 8; i++)
		apic_set_reg(apic, APIC_TMR + 0x10 * i, tmr[i]);
}

Eddie Dong's avatar
Eddie Dong committed
494
495
static void apic_update_ppr(struct kvm_lapic *apic)
{
496
	u32 tpr, isrv, ppr, old_ppr;
Eddie Dong's avatar
Eddie Dong committed
497
498
	int isr;

499
500
	old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI);
	tpr = kvm_apic_get_reg(apic, APIC_TASKPRI);
Eddie Dong's avatar
Eddie Dong committed
501
502
503
504
505
506
507
508
509
510
511
	isr = apic_find_highest_isr(apic);
	isrv = (isr != -1) ? isr : 0;

	if ((tpr & 0xf0) >= (isrv & 0xf0))
		ppr = tpr & 0xff;
	else
		ppr = isrv & 0xf0;

	apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
		   apic, ppr, isr, isrv);

512
513
	if (old_ppr != ppr) {
		apic_set_reg(apic, APIC_PROCPRI, ppr);
514
515
		if (ppr < old_ppr)
			kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
516
	}
Eddie Dong's avatar
Eddie Dong committed
517
518
519
520
521
522
523
524
525
526
}

static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
{
	apic_set_reg(apic, APIC_TASKPRI, tpr);
	apic_update_ppr(apic);
}

int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
{
527
	return dest == 0xff || kvm_apic_id(apic) == dest;
Eddie Dong's avatar
Eddie Dong committed
528
529
530
531
532
}

int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
{
	int result = 0;
Gleb Natapov's avatar
Gleb Natapov committed
533
534
535
	u32 logical_id;

	if (apic_x2apic_mode(apic)) {
536
		logical_id = kvm_apic_get_reg(apic, APIC_LDR);
Gleb Natapov's avatar
Gleb Natapov committed
537
538
		return logical_id & mda;
	}
Eddie Dong's avatar
Eddie Dong committed
539

540
	logical_id = GET_APIC_LOGICAL_ID(kvm_apic_get_reg(apic, APIC_LDR));
Eddie Dong's avatar
Eddie Dong committed
541

542
	switch (kvm_apic_get_reg(apic, APIC_DFR)) {
Eddie Dong's avatar
Eddie Dong committed
543
544
545
546
547
548
549
550
551
552
	case APIC_DFR_FLAT:
		if (logical_id & mda)
			result = 1;
		break;
	case APIC_DFR_CLUSTER:
		if (((logical_id >> 4) == (mda >> 0x4))
		    && (logical_id & mda & 0xf))
			result = 1;
		break;
	default:
553
		apic_debug("Bad DFR vcpu %d: %08x\n",
554
			   apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR));
Eddie Dong's avatar
Eddie Dong committed
555
556
557
558
559
560
		break;
	}

	return result;
}

561
int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
Eddie Dong's avatar
Eddie Dong committed
562
563
564
			   int short_hand, int dest, int dest_mode)
{
	int result = 0;
565
	struct kvm_lapic *target = vcpu->arch.apic;
Eddie Dong's avatar
Eddie Dong committed
566
567

	apic_debug("target %p, source %p, dest 0x%x, "
568
		   "dest_mode 0x%x, short_hand 0x%x\n",
Eddie Dong's avatar
Eddie Dong committed
569
570
		   target, source, dest, dest_mode, short_hand);

Zachary Amsden's avatar
Zachary Amsden committed
571
	ASSERT(target);
Eddie Dong's avatar
Eddie Dong committed
572
573
	switch (short_hand) {
	case APIC_DEST_NOSHORT:
574
		if (dest_mode == 0)
Eddie Dong's avatar
Eddie Dong committed
575
			/* Physical mode. */
576
577
			result = kvm_apic_match_physical_addr(target, dest);
		else
Eddie Dong's avatar
Eddie Dong committed
578
579
580
581
			/* Logical mode. */
			result = kvm_apic_match_logical_addr(target, dest);
		break;
	case APIC_DEST_SELF:
582
		result = (target == source);
Eddie Dong's avatar
Eddie Dong committed
583
584
585
586
587
		break;
	case APIC_DEST_ALLINC:
		result = 1;
		break;
	case APIC_DEST_ALLBUT:
588
		result = (target != source);
Eddie Dong's avatar
Eddie Dong committed
589
590
		break;
	default:
591
592
		apic_debug("kvm: apic: Bad dest shorthand value %x\n",
			   short_hand);
Eddie Dong's avatar
Eddie Dong committed
593
594
595
596
597
598
		break;
	}

	return result;
}

599
bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
600
		struct kvm_lapic_irq *irq, int *r, unsigned long *dest_map)
601
602
603
604
605
606
607
608
609
610
{
	struct kvm_apic_map *map;
	unsigned long bitmap = 1;
	struct kvm_lapic **dst;
	int i;
	bool ret = false;

	*r = -1;

	if (irq->shorthand == APIC_DEST_SELF) {
611
		*r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
		return true;
	}

	if (irq->shorthand)
		return false;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (!map)
		goto out;

	if (irq->dest_mode == 0) { /* physical mode */
		if (irq->delivery_mode == APIC_DM_LOWEST ||
				irq->dest_id == 0xff)
			goto out;
		dst = &map->phys_map[irq->dest_id & 0xff];
	} else {
		u32 mda = irq->dest_id << (32 - map->ldr_bits);

		dst = map->logical_map[apic_cluster_id(map, mda)];

		bitmap = apic_logical_id(map, mda);

		if (irq->delivery_mode == APIC_DM_LOWEST) {
			int l = -1;
			for_each_set_bit(i, &bitmap, 16) {
				if (!dst[i])
					continue;
				if (l < 0)
					l = i;
				else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0)
					l = i;
			}

			bitmap = (l >= 0) ? 1 << l : 0;
		}
	}

	for_each_set_bit(i, &bitmap, 16) {
		if (!dst[i])
			continue;
		if (*r < 0)
			*r = 0;
656
		*r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
657
658
659
660
661
662
663
664
	}

	ret = true;
out:
	rcu_read_unlock();
	return ret;
}

Eddie Dong's avatar
Eddie Dong committed
665
666
667
668
669
/*
 * Add a pending IRQ into lapic.
 * Return 1 if successfully added and 0 if discarded.
 */
static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
670
671
			     int vector, int level, int trig_mode,
			     unsigned long *dest_map)
Eddie Dong's avatar
Eddie Dong committed
672
{
673
	int result = 0;
674
	struct kvm_vcpu *vcpu = apic->vcpu;
Eddie Dong's avatar
Eddie Dong committed
675
676
677

	switch (delivery_mode) {
	case APIC_DM_LOWEST:
678
679
		vcpu->arch.apic_arb_prio++;
	case APIC_DM_FIXED:
Eddie Dong's avatar
Eddie Dong committed
680
681
682
683
		/* FIXME add logic for vcpu on reset */
		if (unlikely(!apic_enabled(apic)))
			break;

684
685
686
		if (dest_map)
			__set_bit(vcpu->vcpu_id, dest_map);

687
688
689
690
691
		if (kvm_x86_ops->deliver_posted_interrupt) {
			result = 1;
			kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
		} else {
			result = !apic_test_and_set_irr(vector, apic);
Eddie Dong's avatar
Eddie Dong committed
692

693
694
695
696
697
698
699
700
701
702
703
704
705
			if (!result) {
				if (trig_mode)
					apic_debug("level trig mode repeatedly "
						"for vector %d", vector);
				goto out;
			}

			kvm_make_request(KVM_REQ_EVENT, vcpu);
			kvm_vcpu_kick(vcpu);
		}
out:
		trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
				trig_mode, vector, !result);
Eddie Dong's avatar
Eddie Dong committed
706
707
708
		break;

	case APIC_DM_REMRD:
709
		apic_debug("Ignoring delivery mode 3\n");
Eddie Dong's avatar
Eddie Dong committed
710
711
712
		break;

	case APIC_DM_SMI:
713
		apic_debug("Ignoring guest SMI\n");
Eddie Dong's avatar
Eddie Dong committed
714
		break;
715

Eddie Dong's avatar
Eddie Dong committed
716
	case APIC_DM_NMI:
717
		result = 1;
718
		kvm_inject_nmi(vcpu);
Jan Kiszka's avatar
Jan Kiszka committed
719
		kvm_vcpu_kick(vcpu);
Eddie Dong's avatar
Eddie Dong committed
720
721
722
		break;

	case APIC_DM_INIT:
723
		if (!trig_mode || level) {
724
			result = 1;
725
726
727
728
729
			/* assumes that there are only KVM_APIC_INIT/SIPI */
			apic->pending_events = (1UL << KVM_APIC_INIT);
			/* make sure pending_events is visible before sending
			 * the request */
			smp_wmb();
730
			kvm_make_request(KVM_REQ_EVENT, vcpu);
731
732
			kvm_vcpu_kick(vcpu);
		} else {
733
734
			apic_debug("Ignoring de-assert INIT to vcpu %d\n",
				   vcpu->vcpu_id);
735
		}
Eddie Dong's avatar
Eddie Dong committed
736
737
738
		break;

	case APIC_DM_STARTUP:
739
740
		apic_debug("SIPI to vcpu %d vector 0x%02x\n",
			   vcpu->vcpu_id, vector);
741
742
743
744
745
746
747
		result = 1;
		apic->sipi_vector = vector;
		/* make sure sipi_vector is visible for the receiver */
		smp_wmb();
		set_bit(KVM_APIC_SIPI, &apic->pending_events);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		kvm_vcpu_kick(vcpu);
Eddie Dong's avatar
Eddie Dong committed
748
749
		break;

750
751
752
753
754
755
756
757
	case APIC_DM_EXTINT:
		/*
		 * Should only be called by kvm_apic_local_deliver() with LVT0,
		 * before NMI watchdog was enabled. Already handled by
		 * kvm_apic_accept_pic_intr().
		 */
		break;

Eddie Dong's avatar
Eddie Dong committed
758
759
760
761
762
763
764
765
	default:
		printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
		       delivery_mode);
		break;
	}
	return result;
}

766
int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
767
{
768
	return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
769
770
}

771
772
773
774
775
776
777
778
779
static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
{
	if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
	    kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
		int trigger_mode;
		if (apic_test_vector(vector, apic->regs + APIC_TMR))
			trigger_mode = IOAPIC_LEVEL_TRIG;
		else
			trigger_mode = IOAPIC_EDGE_TRIG;
780
		kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
781
782
783
	}
}

784
static int apic_set_eoi(struct kvm_lapic *apic)
Eddie Dong's avatar
Eddie Dong committed
785
786
{
	int vector = apic_find_highest_isr(apic);
787
788
789

	trace_kvm_eoi(apic, vector);

Eddie Dong's avatar
Eddie Dong committed
790
791
792
793
794
	/*
	 * Not every write EOI will has corresponding ISR,
	 * one example is when Kernel check timer on setup_IO_APIC
	 */
	if (vector == -1)
795
		return vector;
Eddie Dong's avatar
Eddie Dong committed
796

Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
797
	apic_clear_isr(vector, apic);
Eddie Dong's avatar
Eddie Dong committed
798
799
	apic_update_ppr(apic);

800
	kvm_ioapic_send_eoi(apic, vector);
801
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
802
	return vector;
Eddie Dong's avatar
Eddie Dong committed
803
804
}

805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
/*
 * this interface assumes a trap-like exit, which has already finished
 * desired side effect including vISR and vPPR update.
 */
void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
{
	struct kvm_lapic *apic = vcpu->arch.apic;

	trace_kvm_eoi(apic, vector);

	kvm_ioapic_send_eoi(apic, vector);
	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
}
EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);

Eddie Dong's avatar
Eddie Dong committed
820
821
static void apic_send_ipi(struct kvm_lapic *apic)
{
822
823
	u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR);
	u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2);
824
	struct kvm_lapic_irq irq;
Eddie Dong's avatar
Eddie Dong committed
825

826
827
828
829
830
831
	irq.vector = icr_low & APIC_VECTOR_MASK;
	irq.delivery_mode = icr_low & APIC_MODE_MASK;
	irq.dest_mode = icr_low & APIC_DEST_MASK;
	irq.level = icr_low & APIC_INT_ASSERT;
	irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
	irq.shorthand = icr_low & APIC_SHORT_MASK;
Gleb Natapov's avatar
Gleb Natapov committed
832
833
834
835
	if (apic_x2apic_mode(apic))
		irq.dest_id = icr_high;
	else
		irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
Eddie Dong's avatar
Eddie Dong committed
836

837
838
	trace_kvm_apic_ipi(icr_low, irq.dest_id);

Eddie Dong's avatar
Eddie Dong committed
839
840
841
	apic_debug("icr_high 0x%x, icr_low 0x%x, "
		   "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
		   "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
Glauber Costa's avatar
Glauber Costa committed
842
		   icr_high, icr_low, irq.shorthand, irq.dest_id,
843
844
845
		   irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
		   irq.vector);

846
	kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
Eddie Dong's avatar
Eddie Dong committed
847
848
849
850
}

static u32 apic_get_tmcct(struct kvm_lapic *apic)
{
851
852
	ktime_t remaining;
	s64 ns;
853
	u32 tmcct;
Eddie Dong's avatar
Eddie Dong committed
854
855
856

	ASSERT(apic != NULL);

857
	/* if initial count is 0, current count should also be 0 */
858
	if (kvm_apic_get_reg(apic, APIC_TMICT) == 0)
859
860
		return 0;

861
	remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
862
863
864
	if (ktime_to_ns(remaining) < 0)
		remaining = ktime_set(0, 0);

865
866
867
	ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
	tmcct = div64_u64(ns,
			 (APIC_BUS_CYCLE_NS * apic->divide_count));
Eddie Dong's avatar
Eddie Dong committed
868
869
870
871

	return tmcct;
}

872
873
874
875
876
static void __report_tpr_access(struct kvm_lapic *apic, bool write)
{
	struct kvm_vcpu *vcpu = apic->vcpu;
	struct kvm_run *run = vcpu->run;

877
	kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
878
	run->tpr_access.rip = kvm_rip_read(vcpu);
879
880
881
882
883
884
885
886
887
	run->tpr_access.is_write = write;
}

static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
{
	if (apic->vcpu->arch.tpr_access_reporting)
		__report_tpr_access(apic, write);
}

Eddie Dong's avatar
Eddie Dong committed
888
889
890
891
892
893
894
895
static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
{
	u32 val = 0;

	if (offset >= LAPIC_MMIO_LENGTH)
		return 0;

	switch (offset) {
Gleb Natapov's avatar
Gleb Natapov committed
896
897
898
899
900
901
	case APIC_ID:
		if (apic_x2apic_mode(apic))
			val = kvm_apic_id(apic);
		else
			val = kvm_apic_id(apic) << 24;
		break;
Eddie Dong's avatar
Eddie Dong committed
902
	case APIC_ARBPRI:
903
		apic_debug("Access APIC ARBPRI register which is for P6\n");
Eddie Dong's avatar
Eddie Dong committed
904
905
906
		break;

	case APIC_TMCCT:	/* Timer CCR */
907
908
909
		if (apic_lvtt_tscdeadline(apic))
			return 0;

Eddie Dong's avatar
Eddie Dong committed
910
911
		val = apic_get_tmcct(apic);
		break;
912
913
	case APIC_PROCPRI:
		apic_update_ppr(apic);
914
		val = kvm_apic_get_reg(apic, offset);
915
		break;
916
917
918
	case APIC_TASKPRI:
		report_tpr_access(apic, false);
		/* fall thru */
Eddie Dong's avatar
Eddie Dong committed
919
	default:
920
		val = kvm_apic_get_reg(apic, offset);
Eddie Dong's avatar
Eddie Dong committed
921
922
923
924
925
926
		break;
	}

	return val;
}

Gregory Haskins's avatar
Gregory Haskins committed
927
928
929
930
931
static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
{
	return container_of(dev, struct kvm_lapic, dev);
}

Gleb Natapov's avatar
Gleb Natapov committed
932
933
static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
		void *data)
Eddie Dong's avatar
Eddie Dong committed
934
935
936
{
	unsigned char alignment = offset & 0xf;
	u32 result;
Guo Chao's avatar
Guo Chao committed
937
	/* this bitmask has a bit cleared for each reserved register */
Gleb Natapov's avatar
Gleb Natapov committed
938
	static const u64 rmask = 0x43ff01ffffffe70cULL;
Eddie Dong's avatar
Eddie Dong committed
939
940

	if ((alignment + len) > 4) {
941
942
		apic_debug("KVM_APIC_READ: alignment error %x %d\n",
			   offset, len);
Gleb Natapov's avatar
Gleb Natapov committed
943
		return 1;
Eddie Dong's avatar
Eddie Dong committed
944
	}
Gleb Natapov's avatar
Gleb Natapov committed
945
946

	if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
947
948
		apic_debug("KVM_APIC_READ: read reserved register %x\n",
			   offset);
Gleb Natapov's avatar
Gleb Natapov committed
949
950
951
		return 1;
	}

Eddie Dong's avatar
Eddie Dong committed
952
953
	result = __apic_read(apic, offset & ~0xf);

954
955
	trace_kvm_apic_read(offset, result);

Eddie Dong's avatar
Eddie Dong committed
956
957
958
959
960
961
962
963
964
965
966
	switch (len) {
	case 1:
	case 2:
	case 4:
		memcpy(data, (char *)&result + alignment, len);
		break;
	default:
		printk(KERN_ERR "Local APIC read with len = %x, "
		       "should be 1,2, or 4 instead\n", len);
		break;
	}
967
	return 0;
Eddie Dong's avatar
Eddie Dong committed
968
969
}

Gleb Natapov's avatar
Gleb Natapov committed
970
971
static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
{
972
	return kvm_apic_hw_enabled(apic) &&
Gleb Natapov's avatar
Gleb Natapov committed
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
	    addr >= apic->base_address &&
	    addr < apic->base_address + LAPIC_MMIO_LENGTH;
}

static int apic_mmio_read(struct kvm_io_device *this,
			   gpa_t address, int len, void *data)
{
	struct kvm_lapic *apic = to_lapic(this);
	u32 offset = address - apic->base_address;

	if (!apic_mmio_in_range(apic, address))
		return -EOPNOTSUPP;

	apic_reg_read(apic, offset, len, data);

	return 0;
}

Eddie Dong's avatar
Eddie Dong committed
991
992
993
994
static void update_divide_count(struct kvm_lapic *apic)
{
	u32 tmp1, tmp2, tdcr;

995
	tdcr = kvm_apic_get_reg(apic, APIC_TDCR);
Eddie Dong's avatar
Eddie Dong committed
996
997
	tmp1 = tdcr & 0xf;
	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
998
	apic->divide_count = 0x1 << (tmp2 & 0x7);
Eddie Dong's avatar
Eddie Dong committed
999
1000

	apic_debug("timer divide count is 0x%x\n",