1 | /* $NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $ */ |
2 | |
3 | /* |
4 | * Copyright (c) 2015 Internet Initiative Japan Inc. |
5 | * All rights reserved. |
6 | * |
7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions |
9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. |
15 | * |
16 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
17 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
18 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
19 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
20 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
21 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
22 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
23 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
24 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
25 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
26 | * POSSIBILITY OF SUCH DAMAGE. |
27 | */ |
28 | |
29 | #include <sys/cdefs.h> |
30 | __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $" ); |
31 | |
32 | #include <sys/param.h> |
33 | #include <sys/systm.h> |
34 | #include <sys/kernel.h> |
35 | #include <sys/errno.h> |
36 | #include <sys/cpu.h> |
37 | #include <sys/interrupt.h> |
38 | #include <sys/intr.h> |
39 | #include <sys/kcpuset.h> |
40 | #include <sys/kmem.h> |
41 | #include <sys/proc.h> |
42 | #include <sys/xcall.h> |
43 | #include <sys/sysctl.h> |
44 | |
45 | #include <sys/conf.h> |
46 | #include <sys/intrio.h> |
47 | #include <sys/kauth.h> |
48 | |
49 | #include <machine/limits.h> |
50 | |
51 | #ifdef INTR_DEBUG |
52 | #define DPRINTF(msg) printf msg |
53 | #else |
54 | #define DPRINTF(msg) |
55 | #endif |
56 | |
57 | static struct intrio_set kintrio_set = { "\0" , NULL, 0 }; |
58 | |
59 | #define UNSET_NOINTR_SHIELD 0 |
60 | #define SET_NOINTR_SHIELD 1 |
61 | |
62 | static void |
63 | interrupt_shield_xcall(void *arg1, void *arg2) |
64 | { |
65 | struct cpu_info *ci; |
66 | struct schedstate_percpu *spc; |
67 | int s, shield; |
68 | |
69 | ci = arg1; |
70 | shield = (int)(intptr_t)arg2; |
71 | spc = &ci->ci_schedstate; |
72 | |
73 | s = splsched(); |
74 | if (shield == UNSET_NOINTR_SHIELD) |
75 | spc->spc_flags &= ~SPCF_NOINTR; |
76 | else if (shield == SET_NOINTR_SHIELD) |
77 | spc->spc_flags |= SPCF_NOINTR; |
78 | splx(s); |
79 | } |
80 | |
81 | /* |
82 | * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags. |
83 | */ |
84 | static int |
85 | interrupt_shield(u_int cpu_idx, int shield) |
86 | { |
87 | struct cpu_info *ci; |
88 | struct schedstate_percpu *spc; |
89 | |
90 | KASSERT(mutex_owned(&cpu_lock)); |
91 | |
92 | ci = cpu_lookup(cpu_idx); |
93 | if (ci == NULL) |
94 | return EINVAL; |
95 | |
96 | spc = &ci->ci_schedstate; |
97 | if (shield == UNSET_NOINTR_SHIELD) { |
98 | if ((spc->spc_flags & SPCF_NOINTR) == 0) |
99 | return 0; |
100 | } else if (shield == SET_NOINTR_SHIELD) { |
101 | if ((spc->spc_flags & SPCF_NOINTR) != 0) |
102 | return 0; |
103 | } |
104 | |
105 | if (ci == curcpu() || !mp_online) { |
106 | interrupt_shield_xcall(ci, (void *)(intptr_t)shield); |
107 | } else { |
108 | uint64_t where; |
109 | where = xc_unicast(0, interrupt_shield_xcall, ci, |
110 | (void *)(intptr_t)shield, ci); |
111 | xc_wait(where); |
112 | } |
113 | |
114 | spc->spc_lastmod = time_second; |
115 | return 0; |
116 | } |
117 | |
118 | /* |
119 | * Move all assigned interrupts from "cpu_idx" to the other cpu as possible. |
120 | * The destination cpu is the lowest cpuid of available cpus. |
121 | * If there are no available cpus, give up to move interrupts. |
122 | */ |
123 | static int |
124 | interrupt_avert_intr(u_int cpu_idx) |
125 | { |
126 | kcpuset_t *cpuset; |
127 | struct intrids_handler *ii_handler; |
128 | intrid_t *ids; |
129 | int error, i, nids; |
130 | |
131 | kcpuset_create(&cpuset, true); |
132 | kcpuset_set(cpuset, cpu_idx); |
133 | |
134 | ii_handler = interrupt_construct_intrids(cpuset); |
135 | if (ii_handler == NULL) { |
136 | error = ENOMEM; |
137 | goto out; |
138 | } |
139 | nids = ii_handler->iih_nids; |
140 | if (nids == 0) { |
141 | error = 0; |
142 | goto destruct_out; |
143 | } |
144 | |
145 | interrupt_get_available(cpuset); |
146 | kcpuset_clear(cpuset, cpu_idx); |
147 | if (kcpuset_iszero(cpuset)) { |
148 | DPRINTF(("%s: no available cpu\n" , __func__)); |
149 | error = ENOENT; |
150 | goto destruct_out; |
151 | } |
152 | |
153 | ids = ii_handler->iih_intrids; |
154 | for (i = 0; i < nids; i++) { |
155 | error = interrupt_distribute_handler(ids[i], cpuset, NULL); |
156 | if (error) |
157 | break; |
158 | } |
159 | |
160 | destruct_out: |
161 | interrupt_destruct_intrids(ii_handler); |
162 | out: |
163 | kcpuset_destroy(cpuset); |
164 | return error; |
165 | } |
166 | |
167 | /* |
168 | * Return actual intrio_list_line size. |
169 | * intrio_list_line size is variable by ncpu. |
170 | */ |
171 | static size_t |
172 | interrupt_intrio_list_line_size(void) |
173 | { |
174 | |
175 | return sizeof(struct intrio_list_line) + |
176 | sizeof(struct intrio_list_line_cpu) * (ncpu - 1); |
177 | } |
178 | |
179 | /* |
180 | * Return the size of interrupts list data on success. |
181 | * Reterun 0 on failed. |
182 | */ |
183 | static size_t |
184 | interrupt_intrio_list_size(void) |
185 | { |
186 | struct intrids_handler *ii_handler; |
187 | size_t ilsize; |
188 | |
189 | ilsize = 0; |
190 | |
191 | /* buffer header */ |
192 | ilsize += sizeof(struct intrio_list); |
193 | |
194 | /* il_line body */ |
195 | ii_handler = interrupt_construct_intrids(kcpuset_running); |
196 | if (ii_handler == NULL) |
197 | return 0; |
198 | ilsize += interrupt_intrio_list_line_size() * (ii_handler->iih_nids); |
199 | |
200 | interrupt_destruct_intrids(ii_handler); |
201 | return ilsize; |
202 | } |
203 | |
204 | /* |
205 | * Set intrctl list data to "il", and return list structure bytes. |
206 | * If error occured, return <0. |
207 | * If "data" == NULL, simply return list structure bytes. |
208 | */ |
209 | static int |
210 | interrupt_intrio_list(struct intrio_list *il, int length) |
211 | { |
212 | struct intrio_list_line *illine; |
213 | kcpuset_t *assigned, *avail; |
214 | struct intrids_handler *ii_handler; |
215 | intrid_t *ids; |
216 | size_t ilsize; |
217 | u_int cpu_idx; |
218 | int nids, intr_idx, ret, line_size; |
219 | |
220 | ilsize = interrupt_intrio_list_size(); |
221 | if (ilsize == 0) |
222 | return -ENOMEM; |
223 | |
224 | if (il == NULL) |
225 | return ilsize; |
226 | |
227 | if (length < ilsize) |
228 | return -ENOMEM; |
229 | |
230 | illine = (struct intrio_list_line *) |
231 | ((char *)il + sizeof(struct intrio_list)); |
232 | il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il); |
233 | |
234 | kcpuset_create(&avail, true); |
235 | interrupt_get_available(avail); |
236 | kcpuset_create(&assigned, true); |
237 | |
238 | ii_handler = interrupt_construct_intrids(kcpuset_running); |
239 | if (ii_handler == NULL) { |
240 | DPRINTF(("%s: interrupt_construct_intrids() failed\n" , |
241 | __func__)); |
242 | ret = -ENOMEM; |
243 | goto out; |
244 | } |
245 | |
246 | line_size = interrupt_intrio_list_line_size(); |
247 | /* ensure interrupts are not added after interrupt_intrio_list_size(). */ |
248 | nids = ii_handler->iih_nids; |
249 | ids = ii_handler->iih_intrids; |
250 | if (ilsize < sizeof(struct intrio_list) + line_size * nids) { |
251 | DPRINTF(("%s: interrupts are added during execution.\n" , |
252 | __func__)); |
253 | ret = -ENOMEM; |
254 | goto destruct_out; |
255 | } |
256 | |
257 | for (intr_idx = 0; intr_idx < nids; intr_idx++) { |
258 | char devname[INTRDEVNAMEBUF]; |
259 | |
260 | strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF); |
261 | interrupt_get_devname(ids[intr_idx], devname, sizeof(devname)); |
262 | strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF); |
263 | |
264 | interrupt_get_assigned(ids[intr_idx], assigned); |
265 | for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) { |
266 | struct intrio_list_line_cpu *illcpu = |
267 | &illine->ill_cpu[cpu_idx]; |
268 | |
269 | illcpu->illc_assigned = |
270 | kcpuset_isset(assigned, cpu_idx) ? true : false; |
271 | illcpu->illc_count = |
272 | interrupt_get_count(ids[intr_idx], cpu_idx); |
273 | } |
274 | |
275 | illine = (struct intrio_list_line *) |
276 | ((char *)illine + line_size); |
277 | } |
278 | |
279 | ret = ilsize; |
280 | il->il_version = INTRIO_LIST_VERSION; |
281 | il->il_ncpus = ncpu; |
282 | il->il_nintrs = nids; |
283 | il->il_linesize = line_size; |
284 | il->il_bufsize = ilsize; |
285 | |
286 | destruct_out: |
287 | interrupt_destruct_intrids(ii_handler); |
288 | out: |
289 | kcpuset_destroy(assigned); |
290 | kcpuset_destroy(avail); |
291 | |
292 | return ret; |
293 | } |
294 | |
295 | /* |
296 | * "intrctl list" entry |
297 | */ |
298 | static int |
299 | interrupt_intrio_list_sysctl(SYSCTLFN_ARGS) |
300 | { |
301 | int ret, error; |
302 | void *buf; |
303 | |
304 | if (oldlenp == NULL) |
305 | return EINVAL; |
306 | |
307 | /* |
308 | * If oldp == NULL, the sysctl(8) caller process want to get the size of |
309 | * intrctl list data only. |
310 | */ |
311 | if (oldp == NULL) { |
312 | ret = interrupt_intrio_list(NULL, 0); |
313 | if (ret < 0) |
314 | return -ret; |
315 | |
316 | *oldlenp = ret; |
317 | return 0; |
318 | } |
319 | |
320 | /* |
321 | * If oldp != NULL, the sysctl(8) caller process want to get both the size |
322 | * and the contents of intrctl list data. |
323 | */ |
324 | if (*oldlenp == 0) |
325 | return ENOMEM; |
326 | |
327 | buf = kmem_zalloc(*oldlenp, KM_SLEEP); |
328 | if (buf == NULL) |
329 | return ENOMEM; |
330 | |
331 | ret = interrupt_intrio_list(buf, *oldlenp); |
332 | if (ret < 0) { |
333 | error = -ret; |
334 | goto out; |
335 | } |
336 | error = copyout(buf, oldp, *oldlenp); |
337 | |
338 | out: |
339 | kmem_free(buf, *oldlenp); |
340 | return error; |
341 | } |
342 | |
343 | /* |
344 | * "intrctl affinity" entry |
345 | */ |
346 | static int |
347 | interrupt_set_affinity_sysctl(SYSCTLFN_ARGS) |
348 | { |
349 | struct sysctlnode node; |
350 | struct intrio_set *iset; |
351 | cpuset_t *ucpuset; |
352 | kcpuset_t *kcpuset; |
353 | int error; |
354 | |
355 | error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR, |
356 | KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL); |
357 | if (error) |
358 | return EPERM; |
359 | |
360 | node = *rnode; |
361 | iset = (struct intrio_set *)node.sysctl_data; |
362 | |
363 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
364 | if (error != 0 || newp == NULL) |
365 | return error; |
366 | |
367 | ucpuset = iset->cpuset; |
368 | kcpuset_create(&kcpuset, true); |
369 | error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); |
370 | if (error) |
371 | goto out; |
372 | if (kcpuset_iszero(kcpuset)) { |
373 | error = EINVAL; |
374 | goto out; |
375 | } |
376 | |
377 | error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL); |
378 | |
379 | out: |
380 | kcpuset_destroy(kcpuset); |
381 | return error; |
382 | } |
383 | |
384 | /* |
385 | * "intrctl intr" entry |
386 | */ |
387 | static int |
388 | interrupt_intr_sysctl(SYSCTLFN_ARGS) |
389 | { |
390 | struct sysctlnode node; |
391 | struct intrio_set *iset; |
392 | cpuset_t *ucpuset; |
393 | kcpuset_t *kcpuset; |
394 | int error; |
395 | u_int cpu_idx; |
396 | |
397 | error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, |
398 | KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); |
399 | if (error) |
400 | return EPERM; |
401 | |
402 | node = *rnode; |
403 | iset = (struct intrio_set *)node.sysctl_data; |
404 | |
405 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
406 | if (error != 0 || newp == NULL) |
407 | return error; |
408 | |
409 | ucpuset = iset->cpuset; |
410 | kcpuset_create(&kcpuset, true); |
411 | error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); |
412 | if (error) |
413 | goto out; |
414 | if (kcpuset_iszero(kcpuset)) { |
415 | error = EINVAL; |
416 | goto out; |
417 | } |
418 | |
419 | cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ |
420 | |
421 | mutex_enter(&cpu_lock); |
422 | error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD); |
423 | mutex_exit(&cpu_lock); |
424 | |
425 | out: |
426 | kcpuset_destroy(kcpuset); |
427 | return error; |
428 | } |
429 | |
430 | /* |
431 | * "intrctl nointr" entry |
432 | */ |
433 | static int |
434 | interrupt_nointr_sysctl(SYSCTLFN_ARGS) |
435 | { |
436 | struct sysctlnode node; |
437 | struct intrio_set *iset; |
438 | cpuset_t *ucpuset; |
439 | kcpuset_t *kcpuset; |
440 | int error; |
441 | u_int cpu_idx; |
442 | |
443 | error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, |
444 | KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); |
445 | if (error) |
446 | return EPERM; |
447 | |
448 | node = *rnode; |
449 | iset = (struct intrio_set *)node.sysctl_data; |
450 | |
451 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
452 | if (error != 0 || newp == NULL) |
453 | return error; |
454 | |
455 | ucpuset = iset->cpuset; |
456 | kcpuset_create(&kcpuset, true); |
457 | error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); |
458 | if (error) |
459 | goto out; |
460 | if (kcpuset_iszero(kcpuset)) { |
461 | error = EINVAL; |
462 | goto out; |
463 | } |
464 | |
465 | cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ |
466 | |
467 | mutex_enter(&cpu_lock); |
468 | error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD); |
469 | mutex_exit(&cpu_lock); |
470 | if (error) |
471 | goto out; |
472 | |
473 | error = interrupt_avert_intr(cpu_idx); |
474 | |
475 | out: |
476 | kcpuset_destroy(kcpuset); |
477 | return error; |
478 | } |
479 | |
480 | SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup" ) |
481 | { |
482 | const struct sysctlnode *node = NULL; |
483 | |
484 | sysctl_createv(clog, 0, NULL, &node, |
485 | CTLFLAG_PERMANENT, CTLTYPE_NODE, |
486 | "intr" , SYSCTL_DESCR("Interrupt options" ), |
487 | NULL, 0, NULL, 0, |
488 | CTL_KERN, CTL_CREATE, CTL_EOL); |
489 | |
490 | sysctl_createv(clog, 0, &node, NULL, |
491 | CTLFLAG_PERMANENT, CTLTYPE_STRUCT, |
492 | "list" , SYSCTL_DESCR("intrctl list" ), |
493 | interrupt_intrio_list_sysctl, 0, NULL, |
494 | 0, CTL_CREATE, CTL_EOL); |
495 | |
496 | sysctl_createv(clog, 0, &node, NULL, |
497 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, |
498 | "affinity" , SYSCTL_DESCR("set affinity" ), |
499 | interrupt_set_affinity_sysctl, 0, &kintrio_set, |
500 | sizeof(kintrio_set), CTL_CREATE, CTL_EOL); |
501 | |
502 | sysctl_createv(clog, 0, &node, NULL, |
503 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, |
504 | "intr" , SYSCTL_DESCR("set intr" ), |
505 | interrupt_intr_sysctl, 0, &kintrio_set, |
506 | sizeof(kintrio_set), CTL_CREATE, CTL_EOL); |
507 | |
508 | sysctl_createv(clog, 0, &node, NULL, |
509 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, |
510 | "nointr" , SYSCTL_DESCR("set nointr" ), |
511 | interrupt_nointr_sysctl, 0, &kintrio_set, |
512 | sizeof(kintrio_set), CTL_CREATE, CTL_EOL); |
513 | } |
514 | |