FreeBSD/Linux Kernel Cross Reference
sys/net/netisr.c
1 /*-
2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org>
3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD: releng/5.3/sys/net/netisr.c 136588 2004-10-16 08:43:07Z cvs2svn $
28 */
29
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/rtprio.h>
33 #include <sys/systm.h>
34 #include <sys/interrupt.h>
35 #include <sys/kernel.h>
36 #include <sys/kthread.h>
37 #include <sys/lock.h>
38 #include <sys/malloc.h>
39 #include <sys/proc.h>
40 #include <sys/random.h>
41 #include <sys/resourcevar.h>
42 #include <sys/sysctl.h>
43 #include <sys/unistd.h>
44 #include <machine/atomic.h>
45 #include <machine/cpu.h>
46 #include <machine/stdarg.h>
47
48 #include <sys/mbuf.h>
49 #include <sys/socket.h>
50
51 #include <net/if.h>
52 #include <net/if_types.h>
53 #include <net/if_var.h>
54 #include <net/netisr.h>
55
56 /*
57 * debug_mpsafenet controls network subsystem-wide use of the Giant lock,
58 * from system calls down to interrupt handlers. It can be changed only via
59 * a tunable at boot, not at run-time, due to the complexity of unwinding.
60 * The compiled default is set via a kernel option; right now, the default
61 * unless otherwise specified is to run the network stack without Giant.
62 */
63 #ifdef NET_WITH_GIANT
64 int debug_mpsafenet = 0;
65 #else
66 int debug_mpsafenet = 1;
67 #endif
68 int debug_mpsafenet_toolatetotwiddle = 0;
69
70 TUNABLE_INT("debug.mpsafenet", &debug_mpsafenet);
71 SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0,
72 "Enable/disable MPSAFE network support");
73
74 volatile unsigned int netisr; /* scheduling bits for network */
75
76 struct netisr {
77 netisr_t *ni_handler;
78 struct ifqueue *ni_queue;
79 int ni_flags;
80 } netisrs[32];
81
82 static void *net_ih;
83
84 /*
85 * Not all network code is currently capable of running MPSAFE; however,
86 * most of it is. Since those sections that are not are generally optional
87 * components not shipped with default kernels, we provide a basic way to
88 * determine whether MPSAFE operation is permitted: based on a default of
89 * yes, we permit non-MPSAFE components to use a registration call to
90 * identify that they require Giant. If the system is early in the boot
91 * process still, then we change the debug_mpsafenet setting to choose a
92 * non-MPSAFE execution mode (degraded). If it's too late for that (since
93 * the setting cannot be changed at run time), we generate a console warning
94 * that the configuration may be unsafe.
95 */
96 static int mpsafe_warn_count;
97
98 /*
99 * Function call implementing registration of a non-MPSAFE network component.
100 */
101 void
102 net_warn_not_mpsafe(const char *component)
103 {
104
105 /*
106 * If we're running with Giant over the network stack, there is no
107 * problem.
108 */
109 if (!debug_mpsafenet)
110 return;
111
112 /*
113 * If it's not too late to change the MPSAFE setting for the network
114 * stack, do so now. This effectively suppresses warnings by
115 * components registering later.
116 */
117 if (!debug_mpsafenet_toolatetotwiddle) {
118 debug_mpsafenet = 0;
119 printf("WARNING: debug.mpsafenet forced to 0 as %s requires "
120 "Giant\n", component);
121 return;
122 }
123
124 /*
125 * We must run without Giant, so generate a console warning with some
126 * information with what to do about it. The system may be operating
127 * unsafely, however.
128 */
129 printf("WARNING: Network stack Giant-free, but %s requires Giant.\n",
130 component);
131 if (mpsafe_warn_count == 0)
132 printf(" Consider adding 'options NET_WITH_GIANT' or "
133 "setting debug.mpsafenet=0\n");
134 mpsafe_warn_count++;
135 }
136
137 /*
138 * This sysinit is run after any pre-loaded or compiled-in components have
139 * announced that they require Giant, but before any modules loaded at
140 * run-time.
141 */
142 static void
143 net_mpsafe_toolate(void *arg)
144 {
145
146 debug_mpsafenet_toolatetotwiddle = 1;
147
148 if (!debug_mpsafenet)
149 printf("WARNING: MPSAFE network stack disabled, expect "
150 "reduced performance.\n");
151 }
152
153 SYSINIT(net_mpsafe_toolate, SI_SUB_SETTINGS, SI_ORDER_ANY, net_mpsafe_toolate,
154 NULL);
155
156 void
157 legacy_setsoftnet(void)
158 {
159 swi_sched(net_ih, 0);
160 }
161
162 void
163 netisr_register(int num, netisr_t *handler, struct ifqueue *inq, int flags)
164 {
165
166 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
167 ("bad isr %d", num));
168 netisrs[num].ni_handler = handler;
169 netisrs[num].ni_queue = inq;
170 if ((flags & NETISR_MPSAFE) && !debug_mpsafenet)
171 flags &= ~NETISR_MPSAFE;
172 netisrs[num].ni_flags = flags;
173 }
174
175 void
176 netisr_unregister(int num)
177 {
178 struct netisr *ni;
179
180 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
181 ("bad isr %d", num));
182 ni = &netisrs[num];
183 ni->ni_handler = NULL;
184 if (ni->ni_queue != NULL)
185 IF_DRAIN(ni->ni_queue);
186 ni->ni_queue = NULL;
187 }
188
189 struct isrstat {
190 int isrs_count; /* dispatch count */
191 int isrs_directed; /* ...directly dispatched */
192 int isrs_deferred; /* ...queued instead */
193 int isrs_queued; /* intentionally queueued */
194 int isrs_drop; /* dropped 'cuz no handler */
195 int isrs_swi_count; /* swi_net handlers called */
196 };
197 static struct isrstat isrstat;
198
199 SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters");
200
201 static int netisr_enable = 0;
202 SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW,
203 &netisr_enable, 0, "enable direct dispatch");
204 TUNABLE_INT("net.isr.enable", &netisr_enable);
205
206 SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD,
207 &isrstat.isrs_count, 0, "");
208 SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD,
209 &isrstat.isrs_directed, 0, "");
210 SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD,
211 &isrstat.isrs_deferred, 0, "");
212 SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD,
213 &isrstat.isrs_queued, 0, "");
214 SYSCTL_INT(_net_isr, OID_AUTO, drop, CTLFLAG_RD,
215 &isrstat.isrs_drop, 0, "");
216 SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD,
217 &isrstat.isrs_swi_count, 0, "");
218
219 /*
220 * Process all packets currently present in a netisr queue. Used to
221 * drain an existing set of packets waiting for processing when we
222 * begin direct dispatch, to avoid processing packets out of order.
223 */
224 static void
225 netisr_processqueue(struct netisr *ni)
226 {
227 struct mbuf *m;
228
229 for (;;) {
230 IF_DEQUEUE(ni->ni_queue, m);
231 if (m == NULL)
232 break;
233 ni->ni_handler(m);
234 }
235 }
236
237 /*
238 * Call the netisr directly instead of queueing the packet, if possible.
239 */
240 void
241 netisr_dispatch(int num, struct mbuf *m)
242 {
243 struct netisr *ni;
244
245 isrstat.isrs_count++; /* XXX redundant */
246 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
247 ("bad isr %d", num));
248 ni = &netisrs[num];
249 if (ni->ni_queue == NULL) {
250 isrstat.isrs_drop++;
251 m_freem(m);
252 return;
253 }
254 /*
255 * Do direct dispatch only for MPSAFE netisrs (and
256 * only when enabled). Note that when a netisr is
257 * marked MPSAFE we permit multiple concurrent instances
258 * to run. We guarantee only the order in which
259 * packets are processed for each "dispatch point" in
260 * the system (i.e. call to netisr_dispatch or
261 * netisr_queue). This insures ordering of packets
262 * from an interface but does not guarantee ordering
263 * between multiple places in the system (e.g. IP
264 * dispatched from interfaces vs. IP queued from IPSec).
265 */
266 if (netisr_enable && (ni->ni_flags & NETISR_MPSAFE)) {
267 isrstat.isrs_directed++;
268 /*
269 * NB: We used to drain the queue before handling
270 * the packet but now do not. Doing so here will
271 * not preserve ordering so instead we fallback to
272 * guaranteeing order only from dispatch points
273 * in the system (see above).
274 */
275 ni->ni_handler(m);
276 } else {
277 isrstat.isrs_deferred++;
278 if (IF_HANDOFF(ni->ni_queue, m, NULL))
279 schednetisr(num);
280 }
281 }
282
283 /*
284 * Same as above, but always queue.
285 * This is either used in places where we are not confident that
286 * direct dispatch is possible, or where queueing is required.
287 * It returns (0) on success and ERRNO on failure. On failure the
288 * mbuf has been free'd.
289 */
290 int
291 netisr_queue(int num, struct mbuf *m)
292 {
293 struct netisr *ni;
294
295 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
296 ("bad isr %d", num));
297 ni = &netisrs[num];
298 if (ni->ni_queue == NULL) {
299 isrstat.isrs_drop++;
300 m_freem(m);
301 return (ENXIO);
302 }
303 isrstat.isrs_queued++;
304 if (!IF_HANDOFF(ni->ni_queue, m, NULL))
305 return (ENOBUFS); /* IF_HANDOFF has free'd the mbuf */
306 schednetisr(num);
307 return (0);
308 }
309
310 static void
311 swi_net(void *dummy)
312 {
313 struct netisr *ni;
314 u_int bits;
315 int i;
316 #ifdef DEVICE_POLLING
317 const int polling = 1;
318 #else
319 const int polling = 0;
320 #endif
321
322 do {
323 bits = atomic_readandclear_int(&netisr);
324 if (bits == 0)
325 break;
326 while ((i = ffs(bits)) != 0) {
327 isrstat.isrs_swi_count++;
328 i--;
329 bits &= ~(1 << i);
330 ni = &netisrs[i];
331 if (ni->ni_handler == NULL) {
332 printf("swi_net: unregistered isr %d.\n", i);
333 continue;
334 }
335 if ((ni->ni_flags & NETISR_MPSAFE) == 0) {
336 mtx_lock(&Giant);
337 if (ni->ni_queue == NULL)
338 ni->ni_handler(NULL);
339 else
340 netisr_processqueue(ni);
341 mtx_unlock(&Giant);
342 } else {
343 if (ni->ni_queue == NULL)
344 ni->ni_handler(NULL);
345 else
346 netisr_processqueue(ni);
347 }
348 }
349 } while (polling);
350 }
351
352 static void
353 start_netisr(void *dummy)
354 {
355
356 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, INTR_MPSAFE, &net_ih))
357 panic("start_netisr");
358 }
359 SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL)
Cache object: 84d484c2f52db9931de07af8ad8a9858
|