Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/mips/cavium/octe/ethernet-rx.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /************************************************************************* 2 Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights 3 reserved. 4 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are 8 met: 9 10 * Redistributions of source code must retain the above copyright 11 notice, this list of conditions and the following disclaimer. 12 13 * Redistributions in binary form must reproduce the above 14 copyright notice, this list of conditions and the following 15 disclaimer in the documentation and/or other materials provided 16 with the distribution. 17 18 * Neither the name of Cavium Networks nor the names of 19 its contributors may be used to endorse or promote products 20 derived from this software without specific prior written 21 permission. 22 23 This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. 24 25 TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 26 AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 27 28 *************************************************************************/ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/kernel.h> 38 #include <sys/mbuf.h> 39 #include <sys/socket.h> 40 #include <sys/proc.h> 41 #include <sys/sched.h> 42 #include <sys/smp.h> 43 #include <sys/taskqueue.h> 44 45 #include <net/ethernet.h> 46 #include <net/if.h> 47 #include <net/if_var.h> 48 49 #include "wrapper-cvmx-includes.h" 50 #include "ethernet-headers.h" 51 52 extern int pow_receive_group; 53 extern struct ifnet *cvm_oct_device[]; 54 55 static struct task cvm_oct_task; 56 static struct taskqueue *cvm_oct_taskq; 57 58 static int cvm_oct_rx_active; 59 60 /** 61 * Interrupt handler. The interrupt occurs whenever the POW 62 * transitions from 0->1 packets in our group. 63 * 64 * @param cpl 65 * @param dev_id 66 * @param regs 67 * @return 68 */ 69 int cvm_oct_do_interrupt(void *dev_id) 70 { 71 /* Acknowledge the interrupt */ 72 if (INTERRUPT_LIMIT) 73 cvmx_write_csr(CVMX_POW_WQ_INT, 1<<pow_receive_group); 74 else 75 cvmx_write_csr(CVMX_POW_WQ_INT, 0x10001<<pow_receive_group); 76 77 /* 78 * Schedule task if there isn't one running. 79 */ 80 if (atomic_cmpset_int(&cvm_oct_rx_active, 0, 1)) 81 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 82 83 return FILTER_HANDLED; 84 } 85 86 87 /** 88 * This is called on receive errors, and determines if the packet 89 * can be dropped early-on in cvm_oct_tasklet_rx(). 90 * 91 * @param work Work queue entry pointing to the packet. 92 * @return Non-zero if the packet can be dropped, zero otherwise. 93 */ 94 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work) 95 { 96 if ((work->word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) { 97 /* Ignore length errors on min size packets. Some equipment 98 incorrectly pads packets to 64+4FCS instead of 60+4FCS. 99 Note these packets still get counted as frame errors. */ 100 } else 101 if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) { 102 103 /* We received a packet with either an alignment error or a 104 FCS error. This may be signalling that we are running 105 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the 106 case we need to parse the packet to determine if we can 107 remove a non spec preamble and generate a correct packet */ 108 int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt); 109 int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt); 110 cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl; 111 gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); 112 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { 113 114 uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr); 115 int i = 0; 116 117 while (i < work->word1.s.len-1) { 118 if (*ptr != 0x55) 119 break; 120 ptr++; 121 i++; 122 } 123 124 if (*ptr == 0xd5) { 125 /* 126 DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt); 127 */ 128 work->packet_ptr.s.addr += i+1; 129 work->word1.s.len -= i+5; 130 } else 131 if ((*ptr & 0xf) == 0xd) { 132 /* 133 DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt); 134 */ 135 work->packet_ptr.s.addr += i; 136 work->word1.s.len -= i+4; 137 for (i = 0; i < work->word1.s.len; i++) { 138 *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4); 139 ptr++; 140 } 141 } else { 142 DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt); 143 /* 144 cvmx_helper_dump_packet(work); 145 */ 146 cvm_oct_free_work(work); 147 return 1; 148 } 149 } 150 } else { 151 DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code); 152 cvm_oct_free_work(work); 153 return 1; 154 } 155 156 return 0; 157 } 158 159 /** 160 * Tasklet function that is scheduled on a core when an interrupt occurs. 161 * 162 * @param unused 163 */ 164 void cvm_oct_tasklet_rx(void *context, int pending) 165 { 166 int coreid; 167 uint64_t old_group_mask; 168 int rx_count = 0; 169 int number_to_free; 170 int num_freed; 171 int packet_not_copied; 172 173 sched_pin(); 174 coreid = cvmx_get_core_num(); 175 176 /* Prefetch cvm_oct_device since we know we need it soon */ 177 CVMX_PREFETCH(cvm_oct_device, 0); 178 179 /* Only allow work for our group (and preserve priorities) */ 180 old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); 181 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), 182 (old_group_mask & ~0xFFFFull) | 1<<pow_receive_group); 183 184 while (1) { 185 struct mbuf *m = NULL; 186 int mbuf_in_hw; 187 cvmx_wqe_t *work; 188 189 if ((INTERRUPT_LIMIT == 0) || (rx_count < MAX_RX_PACKETS)) 190 work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT); 191 else 192 work = NULL; 193 CVMX_PREFETCH(work, 0); 194 if (work == NULL) 195 break; 196 197 mbuf_in_hw = work->word2.s.bufs == 1; 198 if ((mbuf_in_hw)) { 199 m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *)); 200 CVMX_PREFETCH(m, offsetof(struct mbuf, m_data)); 201 CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr)); 202 } 203 CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0); 204 //CVMX_PREFETCH(m, 0); 205 206 207 rx_count++; 208 /* Immediately throw away all packets with receive errors */ 209 if ((work->word2.snoip.rcv_error)) { 210 if (cvm_oct_check_rcv_error(work)) 211 continue; 212 } 213 214 /* We can only use the zero copy path if mbufs are in the FPA pool 215 and the packet fits in a single buffer */ 216 if ((mbuf_in_hw)) { 217 CVMX_PREFETCH(m->m_data, 0); 218 219 m->m_pkthdr.len = m->m_len = work->word1.s.len; 220 221 packet_not_copied = 1; 222 223 /* 224 * Adjust the data pointer based on the offset 225 * of the packet within the buffer. 226 */ 227 m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f); 228 } else { 229 230 /* We have to copy the packet. First allocate an 231 mbuf for it */ 232 MGETHDR(m, M_NOWAIT, MT_DATA); 233 if (m == NULL) { 234 DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt); 235 cvm_oct_free_work(work); 236 continue; 237 } 238 239 /* Check if we've received a packet that was entirely 240 stored in the work entry. This is untested */ 241 if ((work->word2.s.bufs == 0)) { 242 uint8_t *ptr = work->packet_data; 243 244 if (cvmx_likely(!work->word2.s.not_IP)) { 245 /* The beginning of the packet moves 246 for IP packets */ 247 if (work->word2.s.is_v6) 248 ptr += 2; 249 else 250 ptr += 6; 251 } 252 panic("%s: not yet implemented; copy in small packet.", __func__); 253 /* No packet buffers to free */ 254 } else { 255 int segments = work->word2.s.bufs; 256 cvmx_buf_ptr_t segment_ptr = work->packet_ptr; 257 int len = work->word1.s.len; 258 259 while (segments--) { 260 cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8); 261 /* Octeon Errata PKI-100: The segment 262 size is wrong. Until it is fixed, 263 calculate the segment size based on 264 the packet pool buffer size. When 265 it is fixed, the following line 266 should be replaced with this one: 267 int segment_size = segment_ptr.s.size; */ 268 int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7)); 269 /* Don't copy more than what is left 270 in the packet */ 271 if (segment_size > len) 272 segment_size = len; 273 /* Copy the data into the packet */ 274 panic("%s: not yet implemented; copy in packet segments.", __func__); 275 #if 0 276 memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size); 277 #endif 278 /* Reduce the amount of bytes left 279 to copy */ 280 len -= segment_size; 281 segment_ptr = next_ptr; 282 } 283 } 284 packet_not_copied = 0; 285 } 286 287 if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) && 288 cvm_oct_device[work->word1.cn38xx.ipprt])) { 289 struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt]; 290 291 /* Only accept packets for devices 292 that are currently up */ 293 if ((ifp->if_flags & IFF_UP)) { 294 m->m_pkthdr.rcvif = ifp; 295 296 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 297 if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error)) 298 m->m_pkthdr.csum_flags = 0; /* XXX */ 299 else { 300 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 301 m->m_pkthdr.csum_data = 0xffff; 302 } 303 } else { 304 m->m_pkthdr.csum_flags = 0; /* XXX */ 305 } 306 307 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 308 309 (*ifp->if_input)(ifp, m); 310 } else { 311 /* Drop any packet received for a device that isn't up */ 312 /* 313 DEBUGPRINT("%s: Device not up, packet dropped\n", 314 if_name(ifp)); 315 */ 316 m_freem(m); 317 } 318 } else { 319 /* Drop any packet received for a device that 320 doesn't exist */ 321 DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt); 322 m_freem(m); 323 } 324 325 /* Check to see if the mbuf and work share 326 the same packet buffer */ 327 if ((packet_not_copied)) { 328 /* This buffer needs to be replaced, increment 329 the number of buffers we need to free by one */ 330 cvmx_fau_atomic_add32( 331 FAU_NUM_PACKET_BUFFERS_TO_FREE, 1); 332 333 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 334 DONT_WRITEBACK(1)); 335 } else 336 cvm_oct_free_work(work); 337 } 338 339 /* 340 * If we hit our limit, schedule another task while we clean up. 341 */ 342 if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) { 343 taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); 344 } else { 345 /* 346 * No more packets, all done. 347 */ 348 if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0)) 349 panic("%s: inconsistent rx active state.", __func__); 350 } 351 352 /* Restore the original POW group mask */ 353 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask); 354 355 /* Refill the packet buffer pool */ 356 number_to_free = 357 cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 358 359 if (number_to_free > 0) { 360 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 361 -number_to_free); 362 num_freed = 363 cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, 364 CVMX_FPA_PACKET_POOL_SIZE, 365 number_to_free); 366 if (num_freed != number_to_free) { 367 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 368 number_to_free - num_freed); 369 } 370 } 371 sched_unpin(); 372 } 373 374 375 376 void cvm_oct_rx_initialize(void) 377 { 378 TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL); 379 380 cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT, 381 taskqueue_thread_enqueue, 382 &cvm_oct_taskq); 383 taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU), 384 PI_NET, "octe taskq"); 385 } 386 387 void cvm_oct_rx_shutdown(void) 388 { 389 panic("%s: not yet implemented.", __func__); 390 } 391
Cache object: b08e231f6597feacc88e33c38d816c80
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]